Merge remote-tracking branch 'airlied/drm-next' into topic/drm-misc

Backmerge latest drm-next to have a baseline for the
s/fence/dma_fence/ patch from Chris.

Signed-off-by: Daniel Vetter <daniel.vetter@intel.com>

182 files changed, 21397 insertions, 3622 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
index 53cf3971dfc3..61360e27715f 100644
--- a/drivers/gpu/drm/amd/amdgpu/Kconfig
+++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
@@ -32,5 +32,4 @@ config DRM_AMDGPU_GART_DEBUGFS
           Selecting this option creates a debugfs file to inspect the mapped
           pages. Uses more memory for housekeeping, enable only for debugging.
 
-source "drivers/gpu/drm/amd/powerplay/Kconfig"
 source "drivers/gpu/drm/amd/acp/Kconfig"
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 9ec262d4b8a2..248a05d02917 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -111,14 +111,10 @@ amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
 amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
 amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
 
-ifneq ($(CONFIG_DRM_AMD_POWERPLAY),)
-
 include $(FULL_AMD_PATH)/powerplay/Makefile
 
 amdgpu-y += $(AMD_POWERPLAY_FILES)
 
-endif
-
 obj-$(CONFIG_DRM_AMDGPU)+= amdgpu.o
 
 CFLAGS_amdgpu_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 235f3902643a..039b57e4644c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1943,6 +1943,7 @@ struct amdgpu_ip_block_status {
         bool valid;
         bool sw;
         bool hw;
+        bool late_initialized;
         bool hang;
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 22c11e7698c8..e3281d4e3e41 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -765,12 +765,20 @@ amdgpu_connector_lvds_detect(struct drm_connector *connector, bool force)
         return ret;
 }
 
-static void amdgpu_connector_destroy(struct drm_connector *connector)
+static void amdgpu_connector_unregister(struct drm_connector *connector)
 {
         struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-        if (amdgpu_connector->ddc_bus->has_aux)
+        if (amdgpu_connector->ddc_bus->has_aux) {
                 drm_dp_aux_unregister(&amdgpu_connector->ddc_bus->aux);
+                amdgpu_connector->ddc_bus->has_aux = false;
+        }
+}
+
+static void amdgpu_connector_destroy(struct drm_connector *connector)
+{
+        struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
+
         amdgpu_connector_free_edid(connector);
         kfree(amdgpu_connector->con_priv);
         drm_connector_unregister(connector);
@@ -824,6 +832,7 @@ static const struct drm_connector_funcs amdgpu_connector_lvds_funcs = {
         .dpms = drm_helper_connector_dpms,
         .detect = amdgpu_connector_lvds_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .early_unregister = amdgpu_connector_unregister,
         .destroy = amdgpu_connector_destroy,
         .set_property = amdgpu_connector_set_lcd_property,
 };
@@ -934,6 +943,7 @@ static const struct drm_connector_funcs amdgpu_connector_vga_funcs = {
         .dpms = drm_helper_connector_dpms,
         .detect = amdgpu_connector_vga_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .early_unregister = amdgpu_connector_unregister,
         .destroy = amdgpu_connector_destroy,
         .set_property = amdgpu_connector_set_property,
 };
@@ -1201,6 +1211,7 @@ static const struct drm_connector_funcs amdgpu_connector_dvi_funcs = {
         .detect = amdgpu_connector_dvi_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = amdgpu_connector_set_property,
+        .early_unregister = amdgpu_connector_unregister,
         .destroy = amdgpu_connector_destroy,
         .force = amdgpu_connector_dvi_force,
 };
@@ -1491,6 +1502,7 @@ static const struct drm_connector_funcs amdgpu_connector_dp_funcs = {
         .detect = amdgpu_connector_dp_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = amdgpu_connector_set_property,
+        .early_unregister = amdgpu_connector_unregister,
         .destroy = amdgpu_connector_destroy,
         .force = amdgpu_connector_dvi_force,
 };
@@ -1500,6 +1512,7 @@ static const struct drm_connector_funcs amdgpu_connector_edp_funcs = {
         .detect = amdgpu_connector_dp_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = amdgpu_connector_set_lcd_property,
+        .early_unregister = amdgpu_connector_unregister,
         .destroy = amdgpu_connector_destroy,
         .force = amdgpu_connector_dvi_force,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index e203e5561107..a5e2fcbef0f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -43,6 +43,9 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
                 ctx->rings[i].sequence = 1;
                 ctx->rings[i].fences = &ctx->fences[amdgpu_sched_jobs * i];
         }
+
+        ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
+
         /* create context entity for each ring */
         for (i = 0; i < adev->num_rings; i++) {
                 struct amdgpu_ring *ring = adev->rings[i];
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 99a15cad6789..b4f4a9239069 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1408,22 +1408,25 @@ static int amdgpu_late_init(struct amdgpu_device *adev)
         for (i = 0; i < adev->num_ip_blocks; i++) {
                 if (!adev->ip_block_status[i].valid)
                         continue;
-                if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_UVD ||
-                        adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_VCE)
-                        continue;
-                /* enable clockgating to save power */
-                r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
-                                                                    AMD_CG_STATE_GATE);
-                if (r) {
-                        DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
-                        return r;
-                }
                 if (adev->ip_blocks[i].funcs->late_init) {
                         r = adev->ip_blocks[i].funcs->late_init((void *)adev);
                         if (r) {
                                 DRM_ERROR("late_init of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
                                 return r;
                         }
+                        adev->ip_block_status[i].late_initialized = true;
+                }
+                /* skip CG for VCE/UVD, it's handled specially */
+                if (adev->ip_blocks[i].type != AMD_IP_BLOCK_TYPE_UVD &&
+                    adev->ip_blocks[i].type != AMD_IP_BLOCK_TYPE_VCE) {
+                        /* enable clockgating to save power */
+                        r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
+                                                                            AMD_CG_STATE_GATE);
+                        if (r) {
+                                DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
+                                          adev->ip_blocks[i].funcs->name, r);
+                                return r;
+                        }
                 }
         }
 
@@ -1434,6 +1437,30 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 {
         int i, r;
 
+        /* need to disable SMC first */
+        for (i = 0; i < adev->num_ip_blocks; i++) {
+                if (!adev->ip_block_status[i].hw)
+                        continue;
+                if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_SMC) {
+                        /* ungate blocks before hw fini so that we can shutdown the blocks safely */
+                        r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
+                                                                            AMD_CG_STATE_UNGATE);
+                        if (r) {
+                                DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
+                                          adev->ip_blocks[i].funcs->name, r);
+                                return r;
+                        }
+                        r = adev->ip_blocks[i].funcs->hw_fini((void *)adev);
+                        /* XXX handle errors */
+                        if (r) {
+                                DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
+                                          adev->ip_blocks[i].funcs->name, r);
+                        }
+                        adev->ip_block_status[i].hw = false;
+                        break;
+                }
+        }
+
         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
                 if (!adev->ip_block_status[i].hw)
                         continue;
@@ -1469,8 +1496,11 @@ static int amdgpu_fini(struct amdgpu_device *adev)
         }
 
         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
+                if (!adev->ip_block_status[i].late_initialized)
+                        continue;
                 if (adev->ip_blocks[i].funcs->late_fini)
                         adev->ip_blocks[i].funcs->late_fini((void *)adev);
+                adev->ip_block_status[i].late_initialized = false;
         }
 
         return 0;
@@ -1822,11 +1852,11 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 
         DRM_INFO("amdgpu: finishing device.\n");
         adev->shutdown = true;
+        drm_crtc_force_disable_all(adev->ddev);
         /* evict vram memory */
         amdgpu_bo_evict_vram(adev);
         amdgpu_ib_pool_fini(adev);
         amdgpu_fence_driver_fini(adev);
-        drm_crtc_force_disable_all(adev->ddev);
         amdgpu_fbdev_fini(adev);
         r = amdgpu_fini(adev);
         kfree(adev->ip_block_status);
@@ -2069,7 +2099,8 @@ static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
                 if (!adev->ip_block_status[i].valid)
                         continue;
                 if (adev->ip_blocks[i].funcs->check_soft_reset)
-                        adev->ip_blocks[i].funcs->check_soft_reset(adev);
+                        adev->ip_block_status[i].hang =
+                                adev->ip_blocks[i].funcs->check_soft_reset(adev);
                 if (adev->ip_block_status[i].hang) {
                         DRM_INFO("IP block:%d is hang!\n", i);
                         asic_hang = true;
@@ -2098,12 +2129,20 @@ static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
 
 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
 {
-        if (adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang ||
-            adev->ip_block_status[AMD_IP_BLOCK_TYPE_SMC].hang ||
-            adev->ip_block_status[AMD_IP_BLOCK_TYPE_ACP].hang ||
-            adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang) {
-                DRM_INFO("Some block need full reset!\n");
-                return true;
+        int i;
+
+        for (i = 0; i < adev->num_ip_blocks; i++) {
+                if (!adev->ip_block_status[i].valid)
+                        continue;
+                if ((adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC) ||
+                    (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_SMC) ||
+                    (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_ACP) ||
+                    (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_DCE)) {
+                        if (adev->ip_block_status[i].hang) {
+                                DRM_INFO("Some block need full reset!\n");
+                                return true;
+                        }
+                }
         }
         return false;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index fe36caf1b7d7..14f57d9915e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -113,24 +113,26 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev,
         printk("\n");
 }
 
+
 u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev)
 {
         struct drm_device *dev = adev->ddev;
         struct drm_crtc *crtc;
         struct amdgpu_crtc *amdgpu_crtc;
-        u32 line_time_us, vblank_lines;
+        u32 vblank_in_pixels;
         u32 vblank_time_us = 0xffffffff; /* if the displays are off, vblank time is max */
 
         if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
                 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                         amdgpu_crtc = to_amdgpu_crtc(crtc);
                         if (crtc->enabled && amdgpu_crtc->enabled && amdgpu_crtc->hw_mode.clock) {
-                                line_time_us = (amdgpu_crtc->hw_mode.crtc_htotal * 1000) /
-                                        amdgpu_crtc->hw_mode.clock;
-                                vblank_lines = amdgpu_crtc->hw_mode.crtc_vblank_end -
+                                vblank_in_pixels =
+                                        amdgpu_crtc->hw_mode.crtc_htotal *
+                                        (amdgpu_crtc->hw_mode.crtc_vblank_end -
                                         amdgpu_crtc->hw_mode.crtc_vdisplay +
-                                        (amdgpu_crtc->v_border * 2);
-                                vblank_time_us = vblank_lines * line_time_us;
+                                        (amdgpu_crtc->v_border * 2));
+
+                                vblank_time_us = vblank_in_pixels * 1000 / amdgpu_crtc->hw_mode.clock;
                                 break;
                         }
                 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index dbe89fb25694..71ed27eb3dde 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -174,7 +174,6 @@ module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444);
 MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)");
 module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
 MODULE_PARM_DESC(powerplay, "Powerplay component (1 = enable, 0 = disable, -1 = auto (default))");
 module_param_named(powerplay, amdgpu_powerplay, int, 0444);
 
@@ -183,7 +182,6 @@ module_param_named(powercontainment, amdgpu_powercontainment, int, 0444);
 
 MODULE_PARM_DESC(ppfeaturemask, "all power features enabled (default))");
 module_param_named(ppfeaturemask, amdgpu_pp_feature_mask, int, 0444);
-#endif
 
 MODULE_PARM_DESC(sclkdeepsleep, "SCLK Deep Sleep (1 = enable (default), 0 = disable)");
 module_param_named(sclkdeepsleep, amdgpu_sclk_deep_sleep_en, int, 0444);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
index 34bab616588c..91d367399956 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
@@ -220,6 +220,7 @@ void amdgpu_i2c_destroy(struct amdgpu_i2c_chan *i2c)
 {
         if (!i2c)
                 return;
+        WARN_ON(i2c->has_aux);
         i2c_del_adapter(&i2c->adapter);
         kfree(i2c);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index 68ad24101a36..7532ff822aa7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -42,7 +42,6 @@ static int amdgpu_powerplay_init(struct amdgpu_device *adev)
         amd_pp = &(adev->powerplay);
 
         if (adev->pp_enabled) {
-#ifdef CONFIG_DRM_AMD_POWERPLAY
                 struct amd_pp_init *pp_init;
 
                 pp_init = kzalloc(sizeof(struct amd_pp_init), GFP_KERNEL);
@@ -55,7 +54,6 @@ static int amdgpu_powerplay_init(struct amdgpu_device *adev)
                 pp_init->device = amdgpu_cgs_create_device(adev);
                 ret = amd_powerplay_init(pp_init, amd_pp);
                 kfree(pp_init);
-#endif
         } else {
                 amd_pp->pp_handle = (void *)adev;
 
@@ -97,7 +95,6 @@ static int amdgpu_pp_early_init(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         int ret = 0;
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
         switch (adev->asic_type) {
         case CHIP_POLARIS11:
         case CHIP_POLARIS10:
@@ -120,9 +117,6 @@ static int amdgpu_pp_early_init(void *handle)
                 adev->pp_enabled = false;
                 break;
         }
-#else
-        adev->pp_enabled = false;
-#endif
 
         ret = amdgpu_powerplay_init(adev);
         if (ret)
@@ -144,12 +138,11 @@ static int amdgpu_pp_late_init(void *handle)
                 ret = adev->powerplay.ip_funcs->late_init(
                                         adev->powerplay.pp_handle);
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
         if (adev->pp_enabled && adev->pm.dpm_enabled) {
                 amdgpu_pm_sysfs_init(adev);
                 amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_COMPLETE_INIT, NULL, NULL);
         }
-#endif
+
         return ret;
 }
 
@@ -162,10 +155,8 @@ static int amdgpu_pp_sw_init(void *handle)
                 ret = adev->powerplay.ip_funcs->sw_init(
                                         adev->powerplay.pp_handle);
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
         if (adev->pp_enabled)
                 adev->pm.dpm_enabled = true;
-#endif
 
         return ret;
 }
@@ -216,7 +207,6 @@ static int amdgpu_pp_hw_fini(void *handle)
 
 static void amdgpu_pp_late_fini(void *handle)
 {
-#ifdef CONFIG_DRM_AMD_POWERPLAY
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev->pp_enabled) {
@@ -227,7 +217,6 @@ static void amdgpu_pp_late_fini(void *handle)
         if (adev->powerplay.ip_funcs->late_fini)
                 adev->powerplay.ip_funcs->late_fini(
                           adev->powerplay.pp_handle);
-#endif
 }
 
 static int amdgpu_pp_suspend(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index e1fa8731d1e2..3cb5e903cd62 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -345,8 +345,8 @@ static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
         ent = debugfs_create_file(name,
                                   S_IFREG | S_IRUGO, root,
                                   ring, &amdgpu_debugfs_ring_fops);
-        if (IS_ERR(ent))
-                return PTR_ERR(ent);
+        if (!ent)
+                return -ENOMEM;
 
         i_size_write(ent->d_inode, ring->ring_size + 12);
         ring->ent = ent;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 887483b8b818..dcaf691f56b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -555,10 +555,13 @@ struct amdgpu_ttm_tt {
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 {
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
+        unsigned int flags = 0;
         unsigned pinned = 0;
         int r;
 
+        if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
+                flags |= FOLL_WRITE;
+
         if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
                 /* check that we only use anonymous memory
                    to prevent problems with writeback */
@@ -581,7 +584,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
                 list_add(&guptask.list, &gtt->guptasks);
                 spin_unlock(&gtt->guptasklock);
 
-                r = get_user_pages(userptr, num_pages, write, 0, p, NULL);
+                r = get_user_pages(userptr, num_pages, flags, p, NULL);
 
                 spin_lock(&gtt->guptasklock);
                 list_del(&guptask.list);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 7a05f79818f1..cb3d252f3c78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -273,7 +273,6 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 
         amdgpu_bo_unreserve(*bo);
 
-        fw_offset = 0;
         for (i = 0; i < AMDGPU_UCODE_ID_MAXIMUM; i++) {
                 ucode = &adev->firmware.ucode[i];
                 if (ucode->fw) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 3b03558ddb01..7fe8fd884f06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -699,6 +699,20 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
                 case 0x05000009: /* clock table */
                         break;
 
+                case 0x0500000c: /* hw config */
+                        switch (p->adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_CIK
+                        case CHIP_KAVERI:
+                        case CHIP_MULLINS:
+#endif
+                        case CHIP_CARRIZO:
+                                break;
+                        default:
+                                r = -EINVAL;
+                                goto out;
+                        }
+                        break;
+
                 case 0x03000001: /* encode */
                         r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 10, idx + 9,
                                                 *size, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index bc4b22c6fc08..06f24322e7c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -878,13 +878,13 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
          * allocation size to the fragment size.
          */
 
-        const uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
+        /* SI and newer are optimized for 64KB */
+        uint64_t frag_flags = AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG);
+        uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
 
         uint64_t frag_start = ALIGN(start, frag_align);
         uint64_t frag_end = end & ~(frag_align - 1);
 
-        uint32_t frag;
-
         /* system pages are non continuously */
         if (params->src || !(flags & AMDGPU_PTE_VALID) ||
             (frag_start >= frag_end)) {
@@ -893,10 +893,6 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
                 return;
         }
 
-        /* use more than 64KB fragment size if possible */
-        frag = lower_32_bits(frag_start | frag_end);
-        frag = likely(frag) ? __ffs(frag) : 31;
-
         /* handle the 4K area at the beginning */
         if (start != frag_start) {
                 amdgpu_vm_update_ptes(params, vm, start, frag_start,
@@ -906,7 +902,7 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
 
         /* handle the area in the middle */
         amdgpu_vm_update_ptes(params, vm, frag_start, frag_end, dst,
-                              flags | AMDGPU_PTE_FRAG(frag));
+                              flags | frag_flags);
 
         /* handle the 4K area at the end */
         if (frag_end != end) {
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
index f80a0834e889..3c082e143730 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
@@ -1514,14 +1514,16 @@ static int cz_dpm_set_powergating_state(void *handle,
         return 0;
 }
 
-/* borrowed from KV, need future unify */
 static int cz_dpm_get_temperature(struct amdgpu_device *adev)
 {
         int actual_temp = 0;
-        uint32_t temp = RREG32_SMC(0xC0300E0C);
+        uint32_t val = RREG32_SMC(ixTHM_TCON_CUR_TMP);
+        uint32_t temp = REG_GET_FIELD(val, THM_TCON_CUR_TMP, CUR_TEMP);
 
-        if (temp)
+        if (REG_GET_FIELD(val, THM_TCON_CUR_TMP, CUR_TEMP_RANGE_SEL))
                 actual_temp = 1000 * ((temp / 8) - 49);
+        else
+                actual_temp = 1000 * (temp / 8);
 
         return actual_temp;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 613ebb7ed50f..4108c686aa7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -3188,16 +3188,11 @@ static int dce_v10_0_wait_for_idle(void *handle)
         return 0;
 }
 
-static int dce_v10_0_check_soft_reset(void *handle)
+static bool dce_v10_0_check_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (dce_v10_0_is_display_hung(adev))
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang = true;
-        else
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang = false;
-
-        return 0;
+        return dce_v10_0_is_display_hung(adev);
 }
 
 static int dce_v10_0_soft_reset(void *handle)
@@ -3205,9 +3200,6 @@ static int dce_v10_0_soft_reset(void *handle)
         u32 srbm_soft_reset = 0, tmp;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang)
-                return 0;
-
         if (dce_v10_0_is_display_hung(adev))
                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 678f5eb6cbc2..f264b8f17ad1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -3159,6 +3159,7 @@ static int dce_v11_0_sw_fini(void *handle)
 
         dce_v11_0_afmt_fini(adev);
 
+        drm_mode_config_cleanup(adev->ddev);
         adev->mode_info.mode_config_initialized = false;
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index a754f2522ba2..c2bd9f045532 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -99,15 +99,15 @@ static void dce_virtual_stop_mc_access(struct amdgpu_device *adev,
                               struct amdgpu_mode_mc_save *save)
 {
         switch (adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_CIK
         case CHIP_BONAIRE:
         case CHIP_HAWAII:
         case CHIP_KAVERI:
         case CHIP_KABINI:
         case CHIP_MULLINS:
-#ifdef CONFIG_DRM_AMDGPU_CIK
                 dce_v8_0_disable_dce(adev);
-#endif
                 break;
+#endif
         case CHIP_FIJI:
         case CHIP_TONGA:
                 dce_v10_0_disable_dce(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 6c6ff57b1c95..ee6a48a09214 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -4087,14 +4087,21 @@ static int gfx_v8_0_rlc_load_microcode(struct amdgpu_device *adev)
 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
 {
         int r;
+        u32 tmp;
 
         gfx_v8_0_rlc_stop(adev);
 
         /* disable CG */
-        WREG32(mmRLC_CGCG_CGLS_CTRL, 0);
+        tmp = RREG32(mmRLC_CGCG_CGLS_CTRL);
+        tmp &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
+                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
+        WREG32(mmRLC_CGCG_CGLS_CTRL, tmp);
         if (adev->asic_type == CHIP_POLARIS11 ||
-            adev->asic_type == CHIP_POLARIS10)
-                WREG32(mmRLC_CGCG_CGLS_CTRL_3D, 0);
+            adev->asic_type == CHIP_POLARIS10) {
+                tmp = RREG32(mmRLC_CGCG_CGLS_CTRL_3D);
+                tmp &= ~0x3;
+                WREG32(mmRLC_CGCG_CGLS_CTRL_3D, tmp);
+        }
 
         /* disable PG */
         WREG32(mmRLC_PG_CNTL, 0);
@@ -5137,7 +5144,7 @@ static int gfx_v8_0_wait_for_idle(void *handle)
         return -ETIMEDOUT;
 }
 
-static int gfx_v8_0_check_soft_reset(void *handle)
+static bool gfx_v8_0_check_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
@@ -5189,16 +5196,14 @@ static int gfx_v8_0_check_soft_reset(void *handle)
                                                 SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
 
         if (grbm_soft_reset || srbm_soft_reset) {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang = true;
                 adev->gfx.grbm_soft_reset = grbm_soft_reset;
                 adev->gfx.srbm_soft_reset = srbm_soft_reset;
+                return true;
         } else {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang = false;
                 adev->gfx.grbm_soft_reset = 0;
                 adev->gfx.srbm_soft_reset = 0;
+                return false;
         }
-
-        return 0;
 }
 
 static void gfx_v8_0_inactive_hqd(struct amdgpu_device *adev,
@@ -5226,7 +5231,8 @@ static int gfx_v8_0_pre_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang)
+        if ((!adev->gfx.grbm_soft_reset) &&
+            (!adev->gfx.srbm_soft_reset))
                 return 0;
 
         grbm_soft_reset = adev->gfx.grbm_soft_reset;
@@ -5264,7 +5270,8 @@ static int gfx_v8_0_soft_reset(void *handle)
         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
         u32 tmp;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang)
+        if ((!adev->gfx.grbm_soft_reset) &&
+            (!adev->gfx.srbm_soft_reset))
                 return 0;
 
         grbm_soft_reset = adev->gfx.grbm_soft_reset;
@@ -5334,7 +5341,8 @@ static int gfx_v8_0_post_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang)
+        if ((!adev->gfx.grbm_soft_reset) &&
+            (!adev->gfx.srbm_soft_reset))
                 return 0;
 
         grbm_soft_reset = adev->gfx.grbm_soft_reset;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 1b319f5bc696..c22ef140a542 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -1099,7 +1099,7 @@ static int gmc_v8_0_wait_for_idle(void *handle)
 
 }
 
-static int gmc_v8_0_check_soft_reset(void *handle)
+static bool gmc_v8_0_check_soft_reset(void *handle)
 {
         u32 srbm_soft_reset = 0;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -1116,20 +1116,19 @@ static int gmc_v8_0_check_soft_reset(void *handle)
                                                         SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
         }
         if (srbm_soft_reset) {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang = true;
                 adev->mc.srbm_soft_reset = srbm_soft_reset;
+                return true;
         } else {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang = false;
                 adev->mc.srbm_soft_reset = 0;
+                return false;
         }
-        return 0;
 }
 
 static int gmc_v8_0_pre_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang)
+        if (!adev->mc.srbm_soft_reset)
                 return 0;
 
         gmc_v8_0_mc_stop(adev, &adev->mc.save);
@@ -1145,7 +1144,7 @@ static int gmc_v8_0_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang)
+        if (!adev->mc.srbm_soft_reset)
                 return 0;
         srbm_soft_reset = adev->mc.srbm_soft_reset;
 
@@ -1175,7 +1174,7 @@ static int gmc_v8_0_post_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang)
+        if (!adev->mc.srbm_soft_reset)
                 return 0;
 
         gmc_v8_0_mc_resume(adev, &adev->mc.save);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index f325fd86430b..a9d10941fb53 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -1268,7 +1268,7 @@ static int sdma_v3_0_wait_for_idle(void *handle)
         return -ETIMEDOUT;
 }
 
-static int sdma_v3_0_check_soft_reset(void *handle)
+static bool sdma_v3_0_check_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset = 0;
@@ -1281,14 +1281,12 @@ static int sdma_v3_0_check_soft_reset(void *handle)
         }
 
         if (srbm_soft_reset) {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang = true;
                 adev->sdma.srbm_soft_reset = srbm_soft_reset;
+                return true;
         } else {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang = false;
                 adev->sdma.srbm_soft_reset = 0;
+                return false;
         }
-
-        return 0;
 }
 
 static int sdma_v3_0_pre_soft_reset(void *handle)
@@ -1296,7 +1294,7 @@ static int sdma_v3_0_pre_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset = 0;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang)
+        if (!adev->sdma.srbm_soft_reset)
                 return 0;
 
         srbm_soft_reset = adev->sdma.srbm_soft_reset;
@@ -1315,7 +1313,7 @@ static int sdma_v3_0_post_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset = 0;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang)
+        if (!adev->sdma.srbm_soft_reset)
                 return 0;
 
         srbm_soft_reset = adev->sdma.srbm_soft_reset;
@@ -1335,7 +1333,7 @@ static int sdma_v3_0_soft_reset(void *handle)
         u32 srbm_soft_reset = 0;
         u32 tmp;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang)
+        if (!adev->sdma.srbm_soft_reset)
                 return 0;
 
         srbm_soft_reset = adev->sdma.srbm_soft_reset;
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 8bd08925b370..3de7bca5854b 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -3499,6 +3499,12 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev,
                 max_sclk = 75000;
                 max_mclk = 80000;
         }
+        /* Limit clocks for some HD8600 parts */
+        if (adev->pdev->device == 0x6660 &&
+            adev->pdev->revision == 0x83) {
+                max_sclk = 75000;
+                max_mclk = 80000;
+        }
 
         if (rps->vce_active) {
                 rps->evclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].evclk;
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index d127d59f953a..b4ea229bb449 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -373,7 +373,7 @@ static int tonga_ih_wait_for_idle(void *handle)
         return -ETIMEDOUT;
 }
 
-static int tonga_ih_check_soft_reset(void *handle)
+static bool tonga_ih_check_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset = 0;
@@ -384,21 +384,19 @@ static int tonga_ih_check_soft_reset(void *handle)
                                                 SOFT_RESET_IH, 1);
 
         if (srbm_soft_reset) {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang = true;
                 adev->irq.srbm_soft_reset = srbm_soft_reset;
+                return true;
         } else {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang = false;
                 adev->irq.srbm_soft_reset = 0;
+                return false;
         }
-
-        return 0;
 }
 
 static int tonga_ih_pre_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang)
+        if (!adev->irq.srbm_soft_reset)
                 return 0;
 
         return tonga_ih_hw_fini(adev);
@@ -408,7 +406,7 @@ static int tonga_ih_post_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang)
+        if (!adev->irq.srbm_soft_reset)
                 return 0;
 
         return tonga_ih_hw_init(adev);
@@ -419,7 +417,7 @@ static int tonga_ih_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang)
+        if (!adev->irq.srbm_soft_reset)
                 return 0;
         srbm_soft_reset = adev->irq.srbm_soft_reset;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index e0fd9f21ed95..ab3df6d75656 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -770,7 +770,7 @@ static int uvd_v6_0_wait_for_idle(void *handle)
 }
 
 #define AMDGPU_UVD_STATUS_BUSY_MASK    0xfd
-static int uvd_v6_0_check_soft_reset(void *handle)
+static bool uvd_v6_0_check_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset = 0;
@@ -782,19 +782,19 @@ static int uvd_v6_0_check_soft_reset(void *handle)
                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
 
         if (srbm_soft_reset) {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang = true;
                 adev->uvd.srbm_soft_reset = srbm_soft_reset;
+                return true;
         } else {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang = false;
                 adev->uvd.srbm_soft_reset = 0;
+                return false;
         }
-        return 0;
 }
+
 static int uvd_v6_0_pre_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang)
+        if (!adev->uvd.srbm_soft_reset)
                 return 0;
 
         uvd_v6_0_stop(adev);
@@ -806,7 +806,7 @@ static int uvd_v6_0_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang)
+        if (!adev->uvd.srbm_soft_reset)
                 return 0;
         srbm_soft_reset = adev->uvd.srbm_soft_reset;
 
@@ -836,7 +836,7 @@ static int uvd_v6_0_post_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang)
+        if (!adev->uvd.srbm_soft_reset)
                 return 0;
 
         mdelay(5);
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 3f6db4ec0102..8533269ec160 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -561,7 +561,7 @@ static int vce_v3_0_wait_for_idle(void *handle)
 #define  AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \
                                       VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK)
 
-static int vce_v3_0_check_soft_reset(void *handle)
+static bool vce_v3_0_check_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset = 0;
@@ -591,16 +591,15 @@ static int vce_v3_0_check_soft_reset(void *handle)
                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
         }
         WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);
+        mutex_unlock(&adev->grbm_idx_mutex);
 
         if (srbm_soft_reset) {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang = true;
                 adev->vce.srbm_soft_reset = srbm_soft_reset;
+                return true;
         } else {
-                adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang = false;
                 adev->vce.srbm_soft_reset = 0;
+                return false;
         }
-        mutex_unlock(&adev->grbm_idx_mutex);
-        return 0;
 }
 
 static int vce_v3_0_soft_reset(void *handle)
@@ -608,7 +607,7 @@ static int vce_v3_0_soft_reset(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 srbm_soft_reset;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang)
+        if (!adev->vce.srbm_soft_reset)
                 return 0;
         srbm_soft_reset = adev->vce.srbm_soft_reset;
 
@@ -638,7 +637,7 @@ static int vce_v3_0_pre_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang)
+        if (!adev->vce.srbm_soft_reset)
                 return 0;
 
         mdelay(5);
@@ -651,7 +650,7 @@ static int vce_v3_0_post_soft_reset(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang)
+        if (!adev->vce.srbm_soft_reset)
                 return 0;
 
         mdelay(5);
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index c934b78c9e2f..bec8125bceb0 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -165,7 +165,7 @@ struct amd_ip_funcs {
         /* poll for idle */
         int (*wait_for_idle)(void *handle);
         /* check soft reset the IP block */
-        int (*check_soft_reset)(void *handle);
+        bool (*check_soft_reset)(void *handle);
         /* pre soft reset the IP block */
         int (*pre_soft_reset)(void *handle);
         /* soft reset the IP block */
diff --git a/drivers/gpu/drm/amd/powerplay/Kconfig b/drivers/gpu/drm/amd/powerplay/Kconfig
deleted file mode 100644
index af380335b425..000000000000
--- a/drivers/gpu/drm/amd/powerplay/Kconfig
+++ /dev/null
@@ -1,6 +0,0 @@
-config DRM_AMD_POWERPLAY
-        bool  "Enable AMD powerplay component"
-        depends on DRM_AMDGPU
-        default n
-        help
-          select this option will enable AMD powerplay component.
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
index 92b117843875..8cee4e0f9fde 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
@@ -49,6 +49,7 @@ static const pem_event_action * const uninitialize_event[] = {
         uninitialize_display_phy_access_tasks,
         disable_gfx_voltage_island_power_gating_tasks,
         disable_gfx_clock_gating_tasks,
+        uninitialize_thermal_controller_tasks,
         set_boot_state_tasks,
         adjust_power_state_tasks,
         disable_dynamic_state_management_tasks,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index 7e4fcbbbe086..960424913496 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -1785,6 +1785,21 @@ static int cz_get_max_high_clocks(struct pp_hwmgr *hwmgr, struct amd_pp_simple_c
         return 0;
 }
 
+static int cz_thermal_get_temperature(struct pp_hwmgr *hwmgr)
+{
+        int actual_temp = 0;
+        uint32_t val = cgs_read_ind_register(hwmgr->device,
+                                             CGS_IND_REG__SMC, ixTHM_TCON_CUR_TMP);
+        uint32_t temp = PHM_GET_FIELD(val, THM_TCON_CUR_TMP, CUR_TEMP);
+
+        if (PHM_GET_FIELD(val, THM_TCON_CUR_TMP, CUR_TEMP_RANGE_SEL))
+                actual_temp = ((temp / 8) - 49) * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+        else
+                actual_temp = (temp / 8) * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+        return actual_temp;
+}
+
 static int cz_read_sensor(struct pp_hwmgr *hwmgr, int idx, int32_t *value)
 {
         struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
@@ -1881,6 +1896,9 @@ static int cz_read_sensor(struct pp_hwmgr *hwmgr, int idx, int32_t *value)
         case AMDGPU_PP_SENSOR_VCE_POWER:
                 *value = cz_hwmgr->vce_power_gated ? 0 : 1;
                 return 0;
+        case AMDGPU_PP_SENSOR_GPU_TEMP:
+                *value = cz_thermal_get_temperature(hwmgr);
+                return 0;
         default:
                 return -EINVAL;
         }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index 508245d49d33..609996c84ad5 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -1030,20 +1030,19 @@ static int smu7_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
         /* disable SCLK dpm */
-        if (!data->sclk_dpm_key_disabled)
-                PP_ASSERT_WITH_CODE(
-                                (smum_send_msg_to_smc(hwmgr->smumgr,
-                                                PPSMC_MSG_DPM_Disable) == 0),
-                                "Failed to disable SCLK DPM!",
-                                return -EINVAL);
+        if (!data->sclk_dpm_key_disabled) {
+                PP_ASSERT_WITH_CODE(true == smum_is_dpm_running(hwmgr),
+                                "Trying to disable SCLK DPM when DPM is disabled",
+                                return 0);
+                smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DPM_Disable);
+        }
 
         /* disable MCLK dpm */
         if (!data->mclk_dpm_key_disabled) {
-                PP_ASSERT_WITH_CODE(
-                                (smum_send_msg_to_smc(hwmgr->smumgr,
-                                                PPSMC_MSG_MCLKDPM_Disable) == 0),
-                                "Failed to disable MCLK DPM!",
-                                return -EINVAL);
+                PP_ASSERT_WITH_CODE(true == smum_is_dpm_running(hwmgr),
+                                "Trying to disable MCLK DPM when DPM is disabled",
+                                return 0);
+                smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_MCLKDPM_Disable);
         }
 
         return 0;
@@ -1069,10 +1068,13 @@ static int smu7_stop_dpm(struct pp_hwmgr *hwmgr)
                                 return -EINVAL);
         }
 
-        if (smu7_disable_sclk_mclk_dpm(hwmgr)) {
-                printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!");
-                return -EINVAL;
-        }
+        smu7_disable_sclk_mclk_dpm(hwmgr);
+
+        PP_ASSERT_WITH_CODE(true == smum_is_dpm_running(hwmgr),
+                        "Trying to disable voltage DPM when DPM is disabled",
+                        return 0);
+
+        smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Voltage_Cntl_Disable);
 
         return 0;
 }
@@ -1226,7 +1228,7 @@ int smu7_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
         PP_ASSERT_WITH_CODE((0 == tmp_result),
                         "Failed to enable VR hot GPIO interrupt!", result = tmp_result);
 
-        smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay);
+        smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_NoDisplay);
 
         tmp_result = smu7_enable_sclk_control(hwmgr);
         PP_ASSERT_WITH_CODE((0 == tmp_result),
@@ -1306,6 +1308,12 @@ int smu7_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
         PP_ASSERT_WITH_CODE((tmp_result == 0),
                         "Failed to disable thermal auto throttle!", result = tmp_result);
 
+        if (1 == PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, FEATURE_STATUS, AVS_ON)) {
+                PP_ASSERT_WITH_CODE((0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableAvfs)),
+                                        "Failed to disable AVFS!",
+                                        return -EINVAL);
+        }
+
         tmp_result = smu7_stop_dpm(hwmgr);
         PP_ASSERT_WITH_CODE((tmp_result == 0),
                         "Failed to stop DPM!", result = tmp_result);
@@ -1452,8 +1460,10 @@ static int smu7_get_evv_voltages(struct pp_hwmgr *hwmgr)
         struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table = NULL;
 
 
-        if (table_info != NULL)
-                sclk_table = table_info->vdd_dep_on_sclk;
+        if (table_info == NULL)
+                return -EINVAL;
+
+        sclk_table = table_info->vdd_dep_on_sclk;
 
         for (i = 0; i < SMU7_MAX_LEAKAGE_COUNT; i++) {
                 vv_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
@@ -3802,13 +3812,15 @@ static inline bool smu7_are_power_levels_equal(const struct smu7_performance_lev
 
 int smu7_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *pstate1, const struct pp_hw_power_state *pstate2, bool *equal)
 {
-        const struct smu7_power_state *psa = cast_const_phw_smu7_power_state(pstate1);
-        const struct smu7_power_state *psb = cast_const_phw_smu7_power_state(pstate2);
+        const struct smu7_power_state *psa;
+        const struct smu7_power_state *psb;
         int i;
 
         if (pstate1 == NULL || pstate2 == NULL || equal == NULL)
                 return -EINVAL;
 
+        psa = cast_const_phw_smu7_power_state(pstate1);
+        psb = cast_const_phw_smu7_power_state(pstate2);
         /* If the two states don't even have the same number of performance levels they cannot be the same state. */
         if (psa->performance_level_count != psb->performance_level_count) {
                 *equal = false;
@@ -4324,6 +4336,7 @@ static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
         .set_mclk_od = smu7_set_mclk_od,
         .get_clock_by_type = smu7_get_clock_by_type,
         .read_sensor = smu7_read_sensor,
+        .dynamic_state_management_disable = smu7_disable_dpm_tasks,
 };
 
 uint8_t smu7_get_sleep_divider_id_from_clock(uint32_t clock,
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
index eda802bc63c8..8c889caba420 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
@@ -2458,7 +2458,7 @@ static int iceland_set_mc_special_registers(struct pp_hwmgr *hwmgr,
                         PP_ASSERT_WITH_CODE((j <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
                                 "Invalid VramInfo table.", return -EINVAL);
 
-                        if (!data->is_memory_gddr5) {
+                        if (!data->is_memory_gddr5 && j < SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE) {
                                 table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
                                 table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
                                 for (k = 0; k < table->num_entries; k++) {
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index 2f58e9e2a59c..a51f8cbcfe26 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -332,17 +332,19 @@ static void armada_drm_crtc_dpms(struct drm_crtc *crtc, int dpms)
 {
         struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
 
-        if (dcrtc->dpms != dpms) {
-                dcrtc->dpms = dpms;
-                if (!IS_ERR(dcrtc->clk) && !dpms_blanked(dpms))
-                        WARN_ON(clk_prepare_enable(dcrtc->clk));
-                armada_drm_crtc_update(dcrtc);
-                if (!IS_ERR(dcrtc->clk) && dpms_blanked(dpms))
-                        clk_disable_unprepare(dcrtc->clk);
+        if (dpms_blanked(dcrtc->dpms) != dpms_blanked(dpms)) {
                 if (dpms_blanked(dpms))
                         armada_drm_vblank_off(dcrtc);
-                else
+                else if (!IS_ERR(dcrtc->clk))
+                        WARN_ON(clk_prepare_enable(dcrtc->clk));
+                dcrtc->dpms = dpms;
+                armada_drm_crtc_update(dcrtc);
+                if (!dpms_blanked(dpms))
                         drm_crtc_vblank_on(&dcrtc->crtc);
+                else if (!IS_ERR(dcrtc->clk))
+                        clk_disable_unprepare(dcrtc->clk);
+        } else if (dcrtc->dpms != dpms) {
+                dcrtc->dpms = dpms;
         }
 }
 
diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
index a5e428d27d2f..806791897304 100644
--- a/drivers/gpu/drm/armada/armada_gem.c
+++ b/drivers/gpu/drm/armada/armada_gem.c
@@ -387,7 +387,7 @@ int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data,
         if (!access_ok(VERIFY_READ, ptr, args->size))
                 return -EFAULT;
 
-        ret = fault_in_multipages_readable(ptr, args->size);
+        ret = fault_in_pages_readable(ptr, args->size);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index 8ea72d821525..800055c39cdb 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -104,8 +104,8 @@ int drm_debugfs_create_files(const struct drm_info_list *files, int count,
                 ent = debugfs_create_file(files[i].name, S_IFREG | S_IRUGO,
                                           root, tmp, &drm_debugfs_fops);
                 if (!ent) {
-                        DRM_ERROR("Cannot create /sys/kernel/debug/dri/%s/%s\n",
-                                  root->d_name.name, files[i].name);
+                        DRM_ERROR("Cannot create /sys/kernel/debug/dri/%pd/%s\n",
+                                  root, files[i].name);
                         kfree(tmp);
                         ret = -1;
                         goto fail;
diff --git a/drivers/gpu/drm/drm_dp_dual_mode_helper.c b/drivers/gpu/drm/drm_dp_dual_mode_helper.c
index a7b2a751f6fe..488355bdafb9 100644
--- a/drivers/gpu/drm/drm_dp_dual_mode_helper.c
+++ b/drivers/gpu/drm/drm_dp_dual_mode_helper.c
@@ -148,6 +148,14 @@ static bool is_type2_adaptor(uint8_t adaptor_id)
                               DP_DUAL_MODE_REV_TYPE2);
 }
 
+static bool is_lspcon_adaptor(const char hdmi_id[DP_DUAL_MODE_HDMI_ID_LEN],
+                              const uint8_t adaptor_id)
+{
+        return is_hdmi_adaptor(hdmi_id) &&
+                (adaptor_id == (DP_DUAL_MODE_TYPE_TYPE2 |
+                 DP_DUAL_MODE_TYPE_HAS_DPCD));
+}
+
 /**
  * drm_dp_dual_mode_detect - Identify the DP dual mode adaptor
  * @adapter: I2C adapter for the DDC bus
@@ -203,6 +211,8 @@ enum drm_dp_dual_mode_type drm_dp_dual_mode_detect(struct i2c_adapter *adapter)
         ret = drm_dp_dual_mode_read(adapter, DP_DUAL_MODE_ADAPTOR_ID,
                                     &adaptor_id, sizeof(adaptor_id));
         if (ret == 0) {
+                if (is_lspcon_adaptor(hdmi_id, adaptor_id))
+                        return DRM_DP_DUAL_MODE_LSPCON;
                 if (is_type2_adaptor(adaptor_id)) {
                         if (is_hdmi_adaptor(hdmi_id))
                                 return DRM_DP_DUAL_MODE_TYPE2_HDMI;
@@ -364,3 +374,96 @@ const char *drm_dp_get_dual_mode_type_name(enum drm_dp_dual_mode_type type)
         }
 }
 EXPORT_SYMBOL(drm_dp_get_dual_mode_type_name);
+
+/**
+ * drm_lspcon_get_mode: Get LSPCON's current mode of operation by
+ * reading offset (0x80, 0x41)
+ * @adapter: I2C-over-aux adapter
+ * @mode: current lspcon mode of operation output variable
+ *
+ * Returns:
+ * 0 on success, sets the current_mode value to appropriate mode
+ * -error on failure
+ */
+int drm_lspcon_get_mode(struct i2c_adapter *adapter,
+                        enum drm_lspcon_mode *mode)
+{
+        u8 data;
+        int ret = 0;
+
+        if (!mode) {
+                DRM_ERROR("NULL input\n");
+                return -EINVAL;
+        }
+
+        /* Read Status: i2c over aux */
+        ret = drm_dp_dual_mode_read(adapter, DP_DUAL_MODE_LSPCON_CURRENT_MODE,
+                                    &data, sizeof(data));
+        if (ret < 0) {
+                DRM_ERROR("LSPCON read(0x80, 0x41) failed\n");
+                return -EFAULT;
+        }
+
+        if (data & DP_DUAL_MODE_LSPCON_MODE_PCON)
+                *mode = DRM_LSPCON_MODE_PCON;
+        else
+                *mode = DRM_LSPCON_MODE_LS;
+        return 0;
+}
+EXPORT_SYMBOL(drm_lspcon_get_mode);
+
+/**
+ * drm_lspcon_set_mode: Change LSPCON's mode of operation by
+ * writing offset (0x80, 0x40)
+ * @adapter: I2C-over-aux adapter
+ * @mode: required mode of operation
+ *
+ * Returns:
+ * 0 on success, -error on failure/timeout
+ */
+int drm_lspcon_set_mode(struct i2c_adapter *adapter,
+                        enum drm_lspcon_mode mode)
+{
+        u8 data = 0;
+        int ret;
+        int time_out = 200;
+        enum drm_lspcon_mode current_mode;
+
+        if (mode == DRM_LSPCON_MODE_PCON)
+                data = DP_DUAL_MODE_LSPCON_MODE_PCON;
+
+        /* Change mode */
+        ret = drm_dp_dual_mode_write(adapter, DP_DUAL_MODE_LSPCON_MODE_CHANGE,
+                                     &data, sizeof(data));
+        if (ret < 0) {
+                DRM_ERROR("LSPCON mode change failed\n");
+                return ret;
+        }
+
+        /*
+         * Confirm mode change by reading the status bit.
+         * Sometimes, it takes a while to change the mode,
+         * so wait and retry until time out or done.
+         */
+        do {
+                ret = drm_lspcon_get_mode(adapter, &current_mode);
+                if (ret) {
+                        DRM_ERROR("can't confirm LSPCON mode change\n");
+                        return ret;
+                } else {
+                        if (current_mode != mode) {
+                                msleep(10);
+                                time_out -= 10;
+                        } else {
+                                DRM_DEBUG_KMS("LSPCON mode changed to %s\n",
+                                                mode == DRM_LSPCON_MODE_LS ?
+                                                "LS" : "PCON");
+                                return 0;
+                        }
+                }
+        } while (time_out);
+
+        DRM_ERROR("LSPCON mode change timed out\n");
+        return -ETIMEDOUT;
+}
+EXPORT_SYMBOL(drm_lspcon_set_mode);
diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
index ac3924a877e0..3e6fe82c6d64 100644
--- a/drivers/gpu/drm/drm_dp_helper.c
+++ b/drivers/gpu/drm/drm_dp_helper.c
@@ -975,6 +975,12 @@ static void unlock_bus(struct i2c_adapter *i2c, unsigned int flags)
         mutex_unlock(&i2c_to_aux(i2c)->hw_mutex);
 }
 
+static const struct i2c_lock_operations drm_dp_i2c_lock_ops = {
+        .lock_bus = lock_bus,
+        .trylock_bus = trylock_bus,
+        .unlock_bus = unlock_bus,
+};
+
 /**
  * drm_dp_aux_init() - minimally initialise an aux channel
  * @aux: DisplayPort AUX channel
@@ -992,9 +998,7 @@ void drm_dp_aux_init(struct drm_dp_aux *aux)
         aux->ddc.algo_data = aux;
         aux->ddc.retries = 3;
 
-        aux->ddc.lock_bus = lock_bus;
-        aux->ddc.trylock_bus = trylock_bus;
-        aux->ddc.unlock_bus = unlock_bus;
+        aux->ddc.lock_ops = &drm_dp_i2c_lock_ops;
 }
 EXPORT_SYMBOL(drm_dp_aux_init);
 
diff --git a/drivers/gpu/drm/drm_info.c b/drivers/gpu/drm/drm_info.c
index 1df2d33d0b40..ffb2ab389d1d 100644
--- a/drivers/gpu/drm/drm_info.c
+++ b/drivers/gpu/drm/drm_info.c
@@ -54,9 +54,6 @@ int drm_name_info(struct seq_file *m, void *data)
 
         mutex_lock(&dev->master_mutex);
         master = dev->master;
-        if (!master)
-                goto out_unlock;
-
         seq_printf(m, "%s", dev->driver->name);
         if (dev->dev)
                 seq_printf(m, " dev=%s", dev_name(dev->dev));
@@ -65,7 +62,6 @@ int drm_name_info(struct seq_file *m, void *data)
         if (dev->unique)
                 seq_printf(m, " unique=%s", dev->unique);
         seq_printf(m, "\n");
-out_unlock:
         mutex_unlock(&dev->master_mutex);
 
         return 0;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
index cb86c7e5495c..d9230132dfbc 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
@@ -329,20 +329,34 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
         /*
          * Append a LINK to the submitted command buffer to return to
          * the ring buffer.  return_target is the ring target address.
-         * We need three dwords: event, wait, link.
+         * We need at most 7 dwords in the return target: 2 cache flush +
+         * 2 semaphore stall + 1 event + 1 wait + 1 link.
          */
-        return_dwords = 3;
+        return_dwords = 7;
         return_target = etnaviv_buffer_reserve(gpu, buffer, return_dwords);
         CMD_LINK(cmdbuf, return_dwords, return_target);
 
         /*
-         * Append event, wait and link pointing back to the wait
-         * command to the ring buffer.
+         * Append a cache flush, stall, event, wait and link pointing back to
+         * the wait command to the ring buffer.
          */
+        if (gpu->exec_state == ETNA_PIPE_2D) {
+                CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_CACHE,
+                                       VIVS_GL_FLUSH_CACHE_PE2D);
+        } else {
+                CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_CACHE,
+                                       VIVS_GL_FLUSH_CACHE_DEPTH |
+                                       VIVS_GL_FLUSH_CACHE_COLOR);
+                CMD_LOAD_STATE(buffer, VIVS_TS_FLUSH_CACHE,
+                                       VIVS_TS_FLUSH_CACHE_FLUSH);
+        }
+        CMD_SEM(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
+        CMD_STALL(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
         CMD_LOAD_STATE(buffer, VIVS_GL_EVENT, VIVS_GL_EVENT_EVENT_ID(event) |
                        VIVS_GL_EVENT_FROM_PE);
         CMD_WAIT(buffer);
-        CMD_LINK(buffer, 2, return_target + 8);
+        CMD_LINK(buffer, 2, etnaviv_iommu_get_cmdbuf_va(gpu, buffer) +
+                            buffer->user_size - 4);
 
         if (drm_debug & DRM_UT_DRIVER)
                 pr_info("stream link to 0x%08x @ 0x%08x %p\n",
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index 9ffca2478e02..3755ef935af4 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -744,19 +744,22 @@ static struct page **etnaviv_gem_userptr_do_get_pages(
         int ret = 0, pinned, npages = etnaviv_obj->base.size >> PAGE_SHIFT;
         struct page **pvec;
         uintptr_t ptr;
+        unsigned int flags = 0;
 
         pvec = drm_malloc_ab(npages, sizeof(struct page *));
         if (!pvec)
                 return ERR_PTR(-ENOMEM);
 
+        if (!etnaviv_obj->userptr.ro)
+                flags |= FOLL_WRITE;
+
         pinned = 0;
         ptr = etnaviv_obj->userptr.ptr;
 
         down_read(&mm->mmap_sem);
         while (pinned < npages) {
                 ret = get_user_pages_remote(task, mm, ptr, npages - pinned,
-                                            !etnaviv_obj->userptr.ro, 0,
-                                            pvec + pinned, NULL);
+                                            flags, pvec + pinned, NULL);
                 if (ret < 0)
                         break;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
index d3796ed8d8c5..169ac96e8f08 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
@@ -330,7 +330,8 @@ u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu,
                         return (u32)buf->vram_node.start;
 
                 mutex_lock(&mmu->lock);
-                ret = etnaviv_iommu_find_iova(mmu, &buf->vram_node, buf->size);
+                ret = etnaviv_iommu_find_iova(mmu, &buf->vram_node,
+                                              buf->size + SZ_64K);
                 if (ret < 0) {
                         mutex_unlock(&mmu->lock);
                         return 0;
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 83f61c513b7e..465d344f3391 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -38,7 +38,6 @@ config DRM_EXYNOS7_DECON
 
 config DRM_EXYNOS_MIXER
         bool "Mixer"
-        depends on !VIDEO_SAMSUNG_S5P_TV
         help
           Choose this option if you want to use Exynos Mixer for DRM.
 
@@ -77,7 +76,7 @@ config DRM_EXYNOS_DP
 
 config DRM_EXYNOS_HDMI
         bool "HDMI"
-        depends on !VIDEO_SAMSUNG_S5P_TV && (DRM_EXYNOS_MIXER || DRM_EXYNOS5433_DECON)
+        depends on DRM_EXYNOS_MIXER || DRM_EXYNOS5433_DECON
         help
           Choose this option if you want to use Exynos HDMI for DRM.
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
index aa92decf4233..fbd13fabdf2d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
@@ -488,7 +488,8 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
                 goto err_free;
         }
 
-        ret = get_vaddr_frames(start, npages, true, true, g2d_userptr->vec);
+        ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
+                g2d_userptr->vec);
         if (ret != npages) {
                 DRM_ERROR("failed to get user pages from userptr.\n");
                 if (ret < 0)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_iommu.h b/drivers/gpu/drm/exynos/exynos_drm_iommu.h
index c8de4913fdbe..87f6b5672e11 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_iommu.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_iommu.h
@@ -66,7 +66,7 @@ static inline int __exynos_iommu_create_mapping(struct exynos_drm_private *priv,
         if (ret)
                 goto free_domain;
 
-        ret = iommu_dma_init_domain(domain, start, size);
+        ret = iommu_dma_init_domain(domain, start, size, NULL);
         if (ret)
                 goto put_cookie;
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
index 3371635cd4d7..b2d5e188b1b8 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
@@ -51,6 +51,7 @@ static void fsl_dcu_drm_disable_crtc(struct drm_crtc *crtc)
                            DCU_MODE_DCU_MODE(DCU_MODE_OFF));
         regmap_write(fsl_dev->regmap, DCU_UPDATE_MODE,
                      DCU_UPDATE_MODE_READREG);
+        clk_disable_unprepare(fsl_dev->pix_clk);
 }
 
 static void fsl_dcu_drm_crtc_enable(struct drm_crtc *crtc)
@@ -58,6 +59,7 @@ static void fsl_dcu_drm_crtc_enable(struct drm_crtc *crtc)
         struct drm_device *dev = crtc->dev;
         struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
 
+        clk_prepare_enable(fsl_dev->pix_clk);
         regmap_update_bits(fsl_dev->regmap, DCU_DCU_MODE,
                            DCU_MODE_DCU_MODE_MASK,
                            DCU_MODE_DCU_MODE(DCU_MODE_NORMAL));
@@ -116,8 +118,6 @@ static void fsl_dcu_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
                      DCU_THRESHOLD_LS_BF_VS(BF_VS_VAL) |
                      DCU_THRESHOLD_OUT_BUF_HIGH(BUF_MAX_VAL) |
                      DCU_THRESHOLD_OUT_BUF_LOW(BUF_MIN_VAL));
-        regmap_write(fsl_dev->regmap, DCU_UPDATE_MODE,
-                     DCU_UPDATE_MODE_READREG);
         return;
 }
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index 0884c45aefe8..e04efbed1a54 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -267,12 +267,8 @@ static int fsl_dcu_drm_pm_resume(struct device *dev)
                 return ret;
         }
 
-        ret = clk_prepare_enable(fsl_dev->pix_clk);
-        if (ret < 0) {
-                dev_err(dev, "failed to enable pix clk\n");
-                goto disable_dcu_clk;
-        }
-
+        if (fsl_dev->tcon)
+                fsl_tcon_bypass_enable(fsl_dev->tcon);
         fsl_dcu_drm_init_planes(fsl_dev->drm);
         drm_atomic_helper_resume(fsl_dev->drm, fsl_dev->state);
 
@@ -284,10 +280,6 @@ static int fsl_dcu_drm_pm_resume(struct device *dev)
         enable_irq(fsl_dev->irq);
 
         return 0;
-
-disable_dcu_clk:
-        clk_disable_unprepare(fsl_dev->clk);
-        return ret;
 }
 #endif
 
@@ -401,18 +393,12 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
                 goto disable_clk;
         }
 
-        ret = clk_prepare_enable(fsl_dev->pix_clk);
-        if (ret < 0) {
-                dev_err(dev, "failed to enable pix clk\n");
-                goto unregister_pix_clk;
-        }
-
         fsl_dev->tcon = fsl_tcon_init(dev);
 
         drm = drm_dev_alloc(driver, dev);
         if (IS_ERR(drm)) {
                 ret = PTR_ERR(drm);
-                goto disable_pix_clk;
+                goto unregister_pix_clk;
         }
 
         fsl_dev->dev = dev;
@@ -433,8 +419,6 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 
 unref:
         drm_dev_unref(drm);
-disable_pix_clk:
-        clk_disable_unprepare(fsl_dev->pix_clk);
 unregister_pix_clk:
         clk_unregister(fsl_dev->pix_clk);
 disable_clk:
@@ -447,7 +431,6 @@ static int fsl_dcu_drm_remove(struct platform_device *pdev)
         struct fsl_dcu_drm_device *fsl_dev = platform_get_drvdata(pdev);
 
         clk_disable_unprepare(fsl_dev->clk);
-        clk_disable_unprepare(fsl_dev->pix_clk);
         clk_unregister(fsl_dev->pix_clk);
         drm_put_dev(fsl_dev->drm);
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index a7e5486bd1e9..9e6f7d8112b3 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -211,11 +211,6 @@ void fsl_dcu_drm_init_planes(struct drm_device *dev)
                 for (j = 1; j <= fsl_dev->soc->layer_regs; j++)
                         regmap_write(fsl_dev->regmap, DCU_CTRLDESCLN(i, j), 0);
         }
-        regmap_update_bits(fsl_dev->regmap, DCU_DCU_MODE,
-                           DCU_MODE_DCU_MODE_MASK,
-                           DCU_MODE_DCU_MODE(DCU_MODE_OFF));
-        regmap_write(fsl_dev->regmap, DCU_UPDATE_MODE,
-                     DCU_UPDATE_MODE_READREG);
 }
 
 struct drm_plane *fsl_dcu_drm_primary_create_plane(struct drm_device *dev)
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
index 26edcc899712..e1dd75b18118 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
@@ -20,38 +20,6 @@
 #include "fsl_dcu_drm_drv.h"
 #include "fsl_tcon.h"
 
-static int
-fsl_dcu_drm_encoder_atomic_check(struct drm_encoder *encoder,
-                                 struct drm_crtc_state *crtc_state,
-                                 struct drm_connector_state *conn_state)
-{
-        return 0;
-}
-
-static void fsl_dcu_drm_encoder_disable(struct drm_encoder *encoder)
-{
-        struct drm_device *dev = encoder->dev;
-        struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
-
-        if (fsl_dev->tcon)
-                fsl_tcon_bypass_disable(fsl_dev->tcon);
-}
-
-static void fsl_dcu_drm_encoder_enable(struct drm_encoder *encoder)
-{
-        struct drm_device *dev = encoder->dev;
-        struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
-
-        if (fsl_dev->tcon)
-                fsl_tcon_bypass_enable(fsl_dev->tcon);
-}
-
-static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
-        .atomic_check = fsl_dcu_drm_encoder_atomic_check,
-        .disable = fsl_dcu_drm_encoder_disable,
-        .enable = fsl_dcu_drm_encoder_enable,
-};
-
 static void fsl_dcu_drm_encoder_destroy(struct drm_encoder *encoder)
 {
         drm_encoder_cleanup(encoder);
@@ -68,13 +36,16 @@ int fsl_dcu_drm_encoder_create(struct fsl_dcu_drm_device *fsl_dev,
         int ret;
 
         encoder->possible_crtcs = 1;
+
+        /* Use bypass mode for parallel RGB/LVDS encoder */
+        if (fsl_dev->tcon)
+                fsl_tcon_bypass_enable(fsl_dev->tcon);
+
         ret = drm_encoder_init(fsl_dev->drm, encoder, &encoder_funcs,
                                DRM_MODE_ENCODER_LVDS, NULL);
         if (ret < 0)
                 return ret;
 
-        drm_encoder_helper_add(encoder, &encoder_helper_funcs);
-
         return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index 7769e469118f..1c1b19ccb92f 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -46,6 +46,31 @@ config DRM_I915_PRELIMINARY_HW_SUPPORT
 
           If in doubt, say "N".
 
+config DRM_I915_CAPTURE_ERROR
+        bool "Enable capturing GPU state following a hang"
+        depends on DRM_I915
+        default y
+        help
+          This option enables capturing the GPU state when a hang is detected.
+          This information is vital for triaging hangs and assists in debugging.
+          Please report any hang to
+            https://bugs.freedesktop.org/enter_bug.cgi?product=DRI
+          for triaging.
+
+          If in doubt, say "Y".
+
+config DRM_I915_COMPRESS_ERROR
+        bool "Compress GPU error state"
+        depends on DRM_I915_CAPTURE_ERROR
+        select ZLIB_DEFLATE
+        default y
+        help
+          This option selects ZLIB_DEFLATE if it isn't already
+          selected and causes any error state captured upon a GPU hang
+          to be compressed using zlib.
+
+          If in doubt, say "Y".
+
 config DRM_I915_USERPTR
         bool "Always enable userptr support"
         depends on DRM_I915
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index a998c2bce70a..612340097f4b 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -42,7 +42,6 @@ i915-y += i915_cmd_parser.o \
           i915_gem_stolen.o \
           i915_gem_tiling.o \
           i915_gem_userptr.o \
-          i915_gpu_error.o \
           i915_trace_points.o \
           intel_breadcrumbs.o \
           intel_engine_cs.o \
@@ -102,11 +101,15 @@ i915-y += dvo_ch7017.o \
           intel_dvo.o \
           intel_hdmi.o \
           intel_i2c.o \
+          intel_lspcon.o \
           intel_lvds.o \
           intel_panel.o \
           intel_sdvo.o \
           intel_tv.o
 
+# Post-mortem debug and GPU hang state capture
+i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o
+
 # virtual gpu code
 i915-y += i915_vgpu.o
 
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index d0f21a6ad60d..34ea4776af70 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -1,5 +1,7 @@
 GVT_DIR := gvt
-GVT_SOURCE := gvt.o
+GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
+        interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \
+        execlist.o scheduler.o sched_policy.o render.o cmd_parser.o
 
 ccflags-y                      += -I$(src) -I$(src)/$(GVT_DIR) -Wall
 i915-y                         += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c
new file mode 100644
index 000000000000..0d41ebc4aea6
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Dexuan Cui
+ *
+ * Contributors:
+ *    Pei Zhang <pei.zhang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Yulei Zhang <yulei.zhang@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define MB_TO_BYTES(mb) ((mb) << 20ULL)
+#define BYTES_TO_MB(b) ((b) >> 20ULL)
+
+#define HOST_LOW_GM_SIZE MB_TO_BYTES(128)
+#define HOST_HIGH_GM_SIZE MB_TO_BYTES(384)
+#define HOST_FENCE 4
+
+static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        u32 alloc_flag, search_flag;
+        u64 start, end, size;
+        struct drm_mm_node *node;
+        int retried = 0;
+        int ret;
+
+        if (high_gm) {
+                search_flag = DRM_MM_SEARCH_BELOW;
+                alloc_flag = DRM_MM_CREATE_TOP;
+                node = &vgpu->gm.high_gm_node;
+                size = vgpu_hidden_sz(vgpu);
+                start = gvt_hidden_gmadr_base(gvt);
+                end = gvt_hidden_gmadr_end(gvt);
+        } else {
+                search_flag = DRM_MM_SEARCH_DEFAULT;
+                alloc_flag = DRM_MM_CREATE_DEFAULT;
+                node = &vgpu->gm.low_gm_node;
+                size = vgpu_aperture_sz(vgpu);
+                start = gvt_aperture_gmadr_base(gvt);
+                end = gvt_aperture_gmadr_end(gvt);
+        }
+
+        mutex_lock(&dev_priv->drm.struct_mutex);
+search_again:
+        ret = drm_mm_insert_node_in_range_generic(&dev_priv->ggtt.base.mm,
+                                                  node, size, 4096, 0,
+                                                  start, end, search_flag,
+                                                  alloc_flag);
+        if (ret) {
+                ret = i915_gem_evict_something(&dev_priv->ggtt.base,
+                                               size, 4096, 0, start, end, 0);
+                if (ret == 0 && ++retried < 3)
+                        goto search_again;
+
+                gvt_err("fail to alloc %s gm space from host, retried %d\n",
+                                high_gm ? "high" : "low", retried);
+        }
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+        return ret;
+}
+
+static int alloc_vgpu_gm(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        int ret;
+
+        ret = alloc_gm(vgpu, false);
+        if (ret)
+                return ret;
+
+        ret = alloc_gm(vgpu, true);
+        if (ret)
+                goto out_free_aperture;
+
+        gvt_dbg_core("vgpu%d: alloc low GM start %llx size %llx\n", vgpu->id,
+                     vgpu_aperture_offset(vgpu), vgpu_aperture_sz(vgpu));
+
+        gvt_dbg_core("vgpu%d: alloc high GM start %llx size %llx\n", vgpu->id,
+                     vgpu_hidden_offset(vgpu), vgpu_hidden_sz(vgpu));
+
+        return 0;
+out_free_aperture:
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        drm_mm_remove_node(&vgpu->gm.low_gm_node);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+        return ret;
+}
+
+static void free_vgpu_gm(struct intel_vgpu *vgpu)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        drm_mm_remove_node(&vgpu->gm.low_gm_node);
+        drm_mm_remove_node(&vgpu->gm.high_gm_node);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+/**
+ * intel_vgpu_write_fence - write fence registers owned by a vGPU
+ * @vgpu: vGPU instance
+ * @fence: vGPU fence register number
+ * @value: Fence register value to be written
+ *
+ * This function is used to write fence registers owned by a vGPU. The vGPU
+ * fence register number will be translated into HW fence register number.
+ *
+ */
+void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
+                u32 fence, u64 value)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        struct drm_i915_fence_reg *reg;
+        i915_reg_t fence_reg_lo, fence_reg_hi;
+
+        assert_rpm_wakelock_held(dev_priv);
+
+        if (WARN_ON(fence > vgpu_fence_sz(vgpu)))
+                return;
+
+        reg = vgpu->fence.regs[fence];
+        if (WARN_ON(!reg))
+                return;
+
+        fence_reg_lo = FENCE_REG_GEN6_LO(reg->id);
+        fence_reg_hi = FENCE_REG_GEN6_HI(reg->id);
+
+        I915_WRITE(fence_reg_lo, 0);
+        POSTING_READ(fence_reg_lo);
+
+        I915_WRITE(fence_reg_hi, upper_32_bits(value));
+        I915_WRITE(fence_reg_lo, lower_32_bits(value));
+        POSTING_READ(fence_reg_lo);
+}
+
+static void free_vgpu_fence(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        struct drm_i915_fence_reg *reg;
+        u32 i;
+
+        if (WARN_ON(!vgpu_fence_sz(vgpu)))
+                return;
+
+        intel_runtime_pm_get(dev_priv);
+
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        for (i = 0; i < vgpu_fence_sz(vgpu); i++) {
+                reg = vgpu->fence.regs[i];
+                intel_vgpu_write_fence(vgpu, i, 0);
+                list_add_tail(&reg->link,
+                              &dev_priv->mm.fence_list);
+        }
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+
+        intel_runtime_pm_put(dev_priv);
+}
+
+static int alloc_vgpu_fence(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        struct drm_i915_fence_reg *reg;
+        int i;
+        struct list_head *pos, *q;
+
+        intel_runtime_pm_get(dev_priv);
+
+        /* Request fences from host */
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        i = 0;
+        list_for_each_safe(pos, q, &dev_priv->mm.fence_list) {
+                reg = list_entry(pos, struct drm_i915_fence_reg, link);
+                if (reg->pin_count || reg->vma)
+                        continue;
+                list_del(pos);
+                vgpu->fence.regs[i] = reg;
+                intel_vgpu_write_fence(vgpu, i, 0);
+                if (++i == vgpu_fence_sz(vgpu))
+                        break;
+        }
+        if (i != vgpu_fence_sz(vgpu))
+                goto out_free_fence;
+
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+        intel_runtime_pm_put(dev_priv);
+        return 0;
+out_free_fence:
+        /* Return fences to host, if fail */
+        for (i = 0; i < vgpu_fence_sz(vgpu); i++) {
+                reg = vgpu->fence.regs[i];
+                if (!reg)
+                        continue;
+                list_add_tail(&reg->link,
+                              &dev_priv->mm.fence_list);
+        }
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+        intel_runtime_pm_put(dev_priv);
+        return -ENOSPC;
+}
+
+static void free_resource(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+
+        gvt->gm.vgpu_allocated_low_gm_size -= vgpu_aperture_sz(vgpu);
+        gvt->gm.vgpu_allocated_high_gm_size -= vgpu_hidden_sz(vgpu);
+        gvt->fence.vgpu_allocated_fence_num -= vgpu_fence_sz(vgpu);
+}
+
+static int alloc_resource(struct intel_vgpu *vgpu,
+                struct intel_vgpu_creation_params *param)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        unsigned long request, avail, max, taken;
+        const char *item;
+
+        if (!param->low_gm_sz || !param->high_gm_sz || !param->fence_sz) {
+                gvt_err("Invalid vGPU creation params\n");
+                return -EINVAL;
+        }
+
+        item = "low GM space";
+        max = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE;
+        taken = gvt->gm.vgpu_allocated_low_gm_size;
+        avail = max - taken;
+        request = MB_TO_BYTES(param->low_gm_sz);
+
+        if (request > avail)
+                goto no_enough_resource;
+
+        vgpu_aperture_sz(vgpu) = request;
+
+        item = "high GM space";
+        max = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE;
+        taken = gvt->gm.vgpu_allocated_high_gm_size;
+        avail = max - taken;
+        request = MB_TO_BYTES(param->high_gm_sz);
+
+        if (request > avail)
+                goto no_enough_resource;
+
+        vgpu_hidden_sz(vgpu) = request;
+
+        item = "fence";
+        max = gvt_fence_sz(gvt) - HOST_FENCE;
+        taken = gvt->fence.vgpu_allocated_fence_num;
+        avail = max - taken;
+        request = param->fence_sz;
+
+        if (request > avail)
+                goto no_enough_resource;
+
+        vgpu_fence_sz(vgpu) = request;
+
+        gvt->gm.vgpu_allocated_low_gm_size += MB_TO_BYTES(param->low_gm_sz);
+        gvt->gm.vgpu_allocated_high_gm_size += MB_TO_BYTES(param->high_gm_sz);
+        gvt->fence.vgpu_allocated_fence_num += param->fence_sz;
+        return 0;
+
+no_enough_resource:
+        gvt_err("vgpu%d: fail to allocate resource %s\n", vgpu->id, item);
+        gvt_err("vgpu%d: request %luMB avail %luMB max %luMB taken %luMB\n",
+                vgpu->id, BYTES_TO_MB(request), BYTES_TO_MB(avail),
+                BYTES_TO_MB(max), BYTES_TO_MB(taken));
+        return -ENOSPC;
+}
+
+/**
+ * inte_gvt_free_vgpu_resource - free HW resource owned by a vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is used to free the HW resource owned by a vGPU.
+ *
+ */
+void intel_vgpu_free_resource(struct intel_vgpu *vgpu)
+{
+        free_vgpu_gm(vgpu);
+        free_vgpu_fence(vgpu);
+        free_resource(vgpu);
+}
+
+/**
+ * intel_alloc_vgpu_resource - allocate HW resource for a vGPU
+ * @vgpu: vGPU
+ * @param: vGPU creation params
+ *
+ * This function is used to allocate HW resource for a vGPU. User specifies
+ * the resource configuration through the creation params.
+ *
+ * Returns:
+ * zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
+                struct intel_vgpu_creation_params *param)
+{
+        int ret;
+
+        ret = alloc_resource(vgpu, param);
+        if (ret)
+                return ret;
+
+        ret = alloc_vgpu_gm(vgpu);
+        if (ret)
+                goto out_free_resource;
+
+        ret = alloc_vgpu_fence(vgpu);
+        if (ret)
+                goto out_free_vgpu_gm;
+
+        return 0;
+
+out_free_vgpu_gm:
+        free_vgpu_gm(vgpu);
+out_free_resource:
+        free_resource(vgpu);
+        return ret;
+}
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
new file mode 100644
index 000000000000..4c687740f5f1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -0,0 +1,288 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+enum {
+        INTEL_GVT_PCI_BAR_GTTMMIO = 0,
+        INTEL_GVT_PCI_BAR_APERTURE,
+        INTEL_GVT_PCI_BAR_PIO,
+        INTEL_GVT_PCI_BAR_MAX,
+};
+
+/**
+ * intel_vgpu_emulate_cfg_read - emulate vGPU configuration space read
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_emulate_cfg_read(void *__vgpu, unsigned int offset,
+        void *p_data, unsigned int bytes)
+{
+        struct intel_vgpu *vgpu = __vgpu;
+
+        if (WARN_ON(bytes > 4))
+                return -EINVAL;
+
+        if (WARN_ON(offset + bytes > INTEL_GVT_MAX_CFG_SPACE_SZ))
+                return -EINVAL;
+
+        memcpy(p_data, vgpu_cfg_space(vgpu) + offset, bytes);
+        return 0;
+}
+
+static int map_aperture(struct intel_vgpu *vgpu, bool map)
+{
+        u64 first_gfn, first_mfn;
+        u64 val;
+        int ret;
+
+        if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked)
+                return 0;
+
+        val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2];
+        if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+                val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
+        else
+                val = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
+
+        first_gfn = (val + vgpu_aperture_offset(vgpu)) >> PAGE_SHIFT;
+        first_mfn = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
+
+        ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn,
+                                                  first_mfn,
+                                                  vgpu_aperture_sz(vgpu)
+                                                  >> PAGE_SHIFT, map,
+                                                  GVT_MAP_APERTURE);
+        if (ret)
+                return ret;
+
+        vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map;
+        return 0;
+}
+
+static int trap_gttmmio(struct intel_vgpu *vgpu, bool trap)
+{
+        u64 start, end;
+        u64 val;
+        int ret;
+
+        if (trap == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].tracked)
+                return 0;
+
+        val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_0];
+        if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+                start = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0);
+        else
+                start = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0);
+
+        start &= ~GENMASK(3, 0);
+        end = start + vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size - 1;
+
+        ret = intel_gvt_hypervisor_set_trap_area(vgpu, start, end, trap);
+        if (ret)
+                return ret;
+
+        vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].tracked = trap;
+        return 0;
+}
+
+static int emulate_pci_command_write(struct intel_vgpu *vgpu,
+        unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u8 old = vgpu_cfg_space(vgpu)[offset];
+        u8 new = *(u8 *)p_data;
+        u8 changed = old ^ new;
+        int ret;
+
+        if (!(changed & PCI_COMMAND_MEMORY))
+                return 0;
+
+        if (old & PCI_COMMAND_MEMORY) {
+                ret = trap_gttmmio(vgpu, false);
+                if (ret)
+                        return ret;
+                ret = map_aperture(vgpu, false);
+                if (ret)
+                        return ret;
+        } else {
+                ret = trap_gttmmio(vgpu, true);
+                if (ret)
+                        return ret;
+                ret = map_aperture(vgpu, true);
+                if (ret)
+                        return ret;
+        }
+
+        memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+        return 0;
+}
+
+static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset,
+        void *p_data, unsigned int bytes)
+{
+        unsigned int bar_index =
+                (rounddown(offset, 8) % PCI_BASE_ADDRESS_0) / 8;
+        u32 new = *(u32 *)(p_data);
+        bool lo = IS_ALIGNED(offset, 8);
+        u64 size;
+        int ret = 0;
+        bool mmio_enabled =
+                vgpu_cfg_space(vgpu)[PCI_COMMAND] & PCI_COMMAND_MEMORY;
+
+        if (WARN_ON(bar_index >= INTEL_GVT_PCI_BAR_MAX))
+                return -EINVAL;
+
+        if (new == 0xffffffff) {
+                /*
+                 * Power-up software can determine how much address
+                 * space the device requires by writing a value of
+                 * all 1's to the register and then reading the value
+                 * back. The device will return 0's in all don't-care
+                 * address bits.
+                 */
+                size = vgpu->cfg_space.bar[bar_index].size;
+                if (lo) {
+                        new = rounddown(new, size);
+                } else {
+                        u32 val = vgpu_cfg_space(vgpu)[rounddown(offset, 8)];
+                        /* for 32bit mode bar it returns all-0 in upper 32
+                         * bit, for 64bit mode bar it will calculate the
+                         * size with lower 32bit and return the corresponding
+                         * value
+                         */
+                        if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+                                new &= (~(size-1)) >> 32;
+                        else
+                                new = 0;
+                }
+                /*
+                 * Unmapp & untrap the BAR, since guest hasn't configured a
+                 * valid GPA
+                 */
+                switch (bar_index) {
+                case INTEL_GVT_PCI_BAR_GTTMMIO:
+                        ret = trap_gttmmio(vgpu, false);
+                        break;
+                case INTEL_GVT_PCI_BAR_APERTURE:
+                        ret = map_aperture(vgpu, false);
+                        break;
+                }
+                intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
+        } else {
+                /*
+                 * Unmapp & untrap the old BAR first, since guest has
+                 * re-configured the BAR
+                 */
+                switch (bar_index) {
+                case INTEL_GVT_PCI_BAR_GTTMMIO:
+                        ret = trap_gttmmio(vgpu, false);
+                        break;
+                case INTEL_GVT_PCI_BAR_APERTURE:
+                        ret = map_aperture(vgpu, false);
+                        break;
+                }
+                intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
+                /* Track the new BAR */
+                if (mmio_enabled) {
+                        switch (bar_index) {
+                        case INTEL_GVT_PCI_BAR_GTTMMIO:
+                                ret = trap_gttmmio(vgpu, true);
+                                break;
+                        case INTEL_GVT_PCI_BAR_APERTURE:
+                                ret = map_aperture(vgpu, true);
+                                break;
+                        }
+                }
+        }
+        return ret;
+}
+
+/**
+ * intel_vgpu_emulate_cfg_read - emulate vGPU configuration space write
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_emulate_cfg_write(void *__vgpu, unsigned int offset,
+        void *p_data, unsigned int bytes)
+{
+        struct intel_vgpu *vgpu = __vgpu;
+        int ret;
+
+        if (WARN_ON(bytes > 4))
+                return -EINVAL;
+
+        if (WARN_ON(offset + bytes >= INTEL_GVT_MAX_CFG_SPACE_SZ))
+                return -EINVAL;
+
+        /* First check if it's PCI_COMMAND */
+        if (IS_ALIGNED(offset, 2) && offset == PCI_COMMAND) {
+                if (WARN_ON(bytes > 2))
+                        return -EINVAL;
+                return emulate_pci_command_write(vgpu, offset, p_data, bytes);
+        }
+
+        switch (rounddown(offset, 4)) {
+        case PCI_BASE_ADDRESS_0:
+        case PCI_BASE_ADDRESS_1:
+        case PCI_BASE_ADDRESS_2:
+        case PCI_BASE_ADDRESS_3:
+                if (WARN_ON(!IS_ALIGNED(offset, 4)))
+                        return -EINVAL;
+                return emulate_pci_bar_write(vgpu, offset, p_data, bytes);
+
+        case INTEL_GVT_PCI_SWSCI:
+                if (WARN_ON(!IS_ALIGNED(offset, 4)))
+                        return -EINVAL;
+                ret = intel_vgpu_emulate_opregion_request(vgpu, *(u32 *)p_data);
+                if (ret)
+                        return ret;
+                break;
+
+        case INTEL_GVT_PCI_OPREGION:
+                if (WARN_ON(!IS_ALIGNED(offset, 4)))
+                        return -EINVAL;
+                ret = intel_vgpu_init_opregion(vgpu, *(u32 *)p_data);
+                if (ret)
+                        return ret;
+
+                memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+                break;
+        default:
+                memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+                break;
+        }
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
new file mode 100644
index 000000000000..aafb57e26288
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -0,0 +1,2831 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Yulei Zhang <yulei.zhang@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include <linux/slab.h>
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+#include "trace.h"
+
+#define INVALID_OP    (~0U)
+
+#define OP_LEN_MI           9
+#define OP_LEN_2D           10
+#define OP_LEN_3D_MEDIA     16
+#define OP_LEN_MFX_VC       16
+#define OP_LEN_VEBOX        16
+
+#define CMD_TYPE(cmd)   (((cmd) >> 29) & 7)
+
+struct sub_op_bits {
+        int hi;
+        int low;
+};
+struct decode_info {
+        char *name;
+        int op_len;
+        int nr_sub_op;
+        struct sub_op_bits *sub_op;
+};
+
+#define   MAX_CMD_BUDGET                        0x7fffffff
+#define   MI_WAIT_FOR_PLANE_C_FLIP_PENDING      (1<<15)
+#define   MI_WAIT_FOR_PLANE_B_FLIP_PENDING      (1<<9)
+#define   MI_WAIT_FOR_PLANE_A_FLIP_PENDING      (1<<1)
+
+#define   MI_WAIT_FOR_SPRITE_C_FLIP_PENDING      (1<<20)
+#define   MI_WAIT_FOR_SPRITE_B_FLIP_PENDING      (1<<10)
+#define   MI_WAIT_FOR_SPRITE_A_FLIP_PENDING      (1<<2)
+
+/* Render Command Map */
+
+/* MI_* command Opcode (28:23) */
+#define OP_MI_NOOP                          0x0
+#define OP_MI_SET_PREDICATE                 0x1  /* HSW+ */
+#define OP_MI_USER_INTERRUPT                0x2
+#define OP_MI_WAIT_FOR_EVENT                0x3
+#define OP_MI_FLUSH                         0x4
+#define OP_MI_ARB_CHECK                     0x5
+#define OP_MI_RS_CONTROL                    0x6  /* HSW+ */
+#define OP_MI_REPORT_HEAD                   0x7
+#define OP_MI_ARB_ON_OFF                    0x8
+#define OP_MI_URB_ATOMIC_ALLOC              0x9  /* HSW+ */
+#define OP_MI_BATCH_BUFFER_END              0xA
+#define OP_MI_SUSPEND_FLUSH                 0xB
+#define OP_MI_PREDICATE                     0xC  /* IVB+ */
+#define OP_MI_TOPOLOGY_FILTER               0xD  /* IVB+ */
+#define OP_MI_SET_APPID                     0xE  /* IVB+ */
+#define OP_MI_RS_CONTEXT                    0xF  /* HSW+ */
+#define OP_MI_LOAD_SCAN_LINES_INCL          0x12 /* HSW+ */
+#define OP_MI_DISPLAY_FLIP                  0x14
+#define OP_MI_SEMAPHORE_MBOX                0x16
+#define OP_MI_SET_CONTEXT                   0x18
+#define OP_MI_MATH                          0x1A
+#define OP_MI_URB_CLEAR                     0x19
+#define OP_MI_SEMAPHORE_SIGNAL              0x1B  /* BDW+ */
+#define OP_MI_SEMAPHORE_WAIT                0x1C  /* BDW+ */
+
+#define OP_MI_STORE_DATA_IMM                0x20
+#define OP_MI_STORE_DATA_INDEX              0x21
+#define OP_MI_LOAD_REGISTER_IMM             0x22
+#define OP_MI_UPDATE_GTT                    0x23
+#define OP_MI_STORE_REGISTER_MEM            0x24
+#define OP_MI_FLUSH_DW                      0x26
+#define OP_MI_CLFLUSH                       0x27
+#define OP_MI_REPORT_PERF_COUNT             0x28
+#define OP_MI_LOAD_REGISTER_MEM             0x29  /* HSW+ */
+#define OP_MI_LOAD_REGISTER_REG             0x2A  /* HSW+ */
+#define OP_MI_RS_STORE_DATA_IMM             0x2B  /* HSW+ */
+#define OP_MI_LOAD_URB_MEM                  0x2C  /* HSW+ */
+#define OP_MI_STORE_URM_MEM                 0x2D  /* HSW+ */
+#define OP_MI_2E                            0x2E  /* BDW+ */
+#define OP_MI_2F                            0x2F  /* BDW+ */
+#define OP_MI_BATCH_BUFFER_START            0x31
+
+/* Bit definition for dword 0 */
+#define _CMDBIT_BB_START_IN_PPGTT       (1UL << 8)
+
+#define OP_MI_CONDITIONAL_BATCH_BUFFER_END  0x36
+
+#define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2))
+#define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U))
+#define BATCH_BUFFER_ADR_SPACE_BIT(x)   (((x) >> 8) & 1U)
+#define BATCH_BUFFER_2ND_LEVEL_BIT(x)   ((x) >> 22 & 1U)
+
+/* 2D command: Opcode (28:22) */
+#define OP_2D(x)    ((2<<7) | x)
+
+#define OP_XY_SETUP_BLT                             OP_2D(0x1)
+#define OP_XY_SETUP_CLIP_BLT                        OP_2D(0x3)
+#define OP_XY_SETUP_MONO_PATTERN_SL_BLT             OP_2D(0x11)
+#define OP_XY_PIXEL_BLT                             OP_2D(0x24)
+#define OP_XY_SCANLINES_BLT                         OP_2D(0x25)
+#define OP_XY_TEXT_BLT                              OP_2D(0x26)
+#define OP_XY_TEXT_IMMEDIATE_BLT                    OP_2D(0x31)
+#define OP_XY_COLOR_BLT                             OP_2D(0x50)
+#define OP_XY_PAT_BLT                               OP_2D(0x51)
+#define OP_XY_MONO_PAT_BLT                          OP_2D(0x52)
+#define OP_XY_SRC_COPY_BLT                          OP_2D(0x53)
+#define OP_XY_MONO_SRC_COPY_BLT                     OP_2D(0x54)
+#define OP_XY_FULL_BLT                              OP_2D(0x55)
+#define OP_XY_FULL_MONO_SRC_BLT                     OP_2D(0x56)
+#define OP_XY_FULL_MONO_PATTERN_BLT                 OP_2D(0x57)
+#define OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT        OP_2D(0x58)
+#define OP_XY_MONO_PAT_FIXED_BLT                    OP_2D(0x59)
+#define OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT           OP_2D(0x71)
+#define OP_XY_PAT_BLT_IMMEDIATE                     OP_2D(0x72)
+#define OP_XY_SRC_COPY_CHROMA_BLT                   OP_2D(0x73)
+#define OP_XY_FULL_IMMEDIATE_PATTERN_BLT            OP_2D(0x74)
+#define OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT   OP_2D(0x75)
+#define OP_XY_PAT_CHROMA_BLT                        OP_2D(0x76)
+#define OP_XY_PAT_CHROMA_BLT_IMMEDIATE              OP_2D(0x77)
+
+/* 3D/Media Command: Pipeline Type(28:27) Opcode(26:24) Sub Opcode(23:16) */
+#define OP_3D_MEDIA(sub_type, opcode, sub_opcode) \
+        ((3 << 13) | ((sub_type) << 11) | ((opcode) << 8) | (sub_opcode))
+
+#define OP_STATE_PREFETCH                       OP_3D_MEDIA(0x0, 0x0, 0x03)
+
+#define OP_STATE_BASE_ADDRESS                   OP_3D_MEDIA(0x0, 0x1, 0x01)
+#define OP_STATE_SIP                            OP_3D_MEDIA(0x0, 0x1, 0x02)
+#define OP_3D_MEDIA_0_1_4                       OP_3D_MEDIA(0x0, 0x1, 0x04)
+
+#define OP_3DSTATE_VF_STATISTICS_GM45           OP_3D_MEDIA(0x1, 0x0, 0x0B)
+
+#define OP_PIPELINE_SELECT                      OP_3D_MEDIA(0x1, 0x1, 0x04)
+
+#define OP_MEDIA_VFE_STATE                      OP_3D_MEDIA(0x2, 0x0, 0x0)
+#define OP_MEDIA_CURBE_LOAD                     OP_3D_MEDIA(0x2, 0x0, 0x1)
+#define OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD      OP_3D_MEDIA(0x2, 0x0, 0x2)
+#define OP_MEDIA_GATEWAY_STATE                  OP_3D_MEDIA(0x2, 0x0, 0x3)
+#define OP_MEDIA_STATE_FLUSH                    OP_3D_MEDIA(0x2, 0x0, 0x4)
+
+#define OP_MEDIA_OBJECT                         OP_3D_MEDIA(0x2, 0x1, 0x0)
+#define OP_MEDIA_OBJECT_PRT                     OP_3D_MEDIA(0x2, 0x1, 0x2)
+#define OP_MEDIA_OBJECT_WALKER                  OP_3D_MEDIA(0x2, 0x1, 0x3)
+#define OP_GPGPU_WALKER                         OP_3D_MEDIA(0x2, 0x1, 0x5)
+
+#define OP_3DSTATE_CLEAR_PARAMS                 OP_3D_MEDIA(0x3, 0x0, 0x04) /* IVB+ */
+#define OP_3DSTATE_DEPTH_BUFFER                 OP_3D_MEDIA(0x3, 0x0, 0x05) /* IVB+ */
+#define OP_3DSTATE_STENCIL_BUFFER               OP_3D_MEDIA(0x3, 0x0, 0x06) /* IVB+ */
+#define OP_3DSTATE_HIER_DEPTH_BUFFER            OP_3D_MEDIA(0x3, 0x0, 0x07) /* IVB+ */
+#define OP_3DSTATE_VERTEX_BUFFERS               OP_3D_MEDIA(0x3, 0x0, 0x08)
+#define OP_3DSTATE_VERTEX_ELEMENTS              OP_3D_MEDIA(0x3, 0x0, 0x09)
+#define OP_3DSTATE_INDEX_BUFFER                 OP_3D_MEDIA(0x3, 0x0, 0x0A)
+#define OP_3DSTATE_VF_STATISTICS                OP_3D_MEDIA(0x3, 0x0, 0x0B)
+#define OP_3DSTATE_VF                           OP_3D_MEDIA(0x3, 0x0, 0x0C)  /* HSW+ */
+#define OP_3DSTATE_CC_STATE_POINTERS            OP_3D_MEDIA(0x3, 0x0, 0x0E)
+#define OP_3DSTATE_SCISSOR_STATE_POINTERS       OP_3D_MEDIA(0x3, 0x0, 0x0F)
+#define OP_3DSTATE_VS                           OP_3D_MEDIA(0x3, 0x0, 0x10)
+#define OP_3DSTATE_GS                           OP_3D_MEDIA(0x3, 0x0, 0x11)
+#define OP_3DSTATE_CLIP                         OP_3D_MEDIA(0x3, 0x0, 0x12)
+#define OP_3DSTATE_SF                           OP_3D_MEDIA(0x3, 0x0, 0x13)
+#define OP_3DSTATE_WM                           OP_3D_MEDIA(0x3, 0x0, 0x14)
+#define OP_3DSTATE_CONSTANT_VS                  OP_3D_MEDIA(0x3, 0x0, 0x15)
+#define OP_3DSTATE_CONSTANT_GS                  OP_3D_MEDIA(0x3, 0x0, 0x16)
+#define OP_3DSTATE_CONSTANT_PS                  OP_3D_MEDIA(0x3, 0x0, 0x17)
+#define OP_3DSTATE_SAMPLE_MASK                  OP_3D_MEDIA(0x3, 0x0, 0x18)
+#define OP_3DSTATE_CONSTANT_HS                  OP_3D_MEDIA(0x3, 0x0, 0x19) /* IVB+ */
+#define OP_3DSTATE_CONSTANT_DS                  OP_3D_MEDIA(0x3, 0x0, 0x1A) /* IVB+ */
+#define OP_3DSTATE_HS                           OP_3D_MEDIA(0x3, 0x0, 0x1B) /* IVB+ */
+#define OP_3DSTATE_TE                           OP_3D_MEDIA(0x3, 0x0, 0x1C) /* IVB+ */
+#define OP_3DSTATE_DS                           OP_3D_MEDIA(0x3, 0x0, 0x1D) /* IVB+ */
+#define OP_3DSTATE_STREAMOUT                    OP_3D_MEDIA(0x3, 0x0, 0x1E) /* IVB+ */
+#define OP_3DSTATE_SBE                          OP_3D_MEDIA(0x3, 0x0, 0x1F) /* IVB+ */
+#define OP_3DSTATE_PS                           OP_3D_MEDIA(0x3, 0x0, 0x20) /* IVB+ */
+#define OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP OP_3D_MEDIA(0x3, 0x0, 0x21) /* IVB+ */
+#define OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC   OP_3D_MEDIA(0x3, 0x0, 0x23) /* IVB+ */
+#define OP_3DSTATE_BLEND_STATE_POINTERS         OP_3D_MEDIA(0x3, 0x0, 0x24) /* IVB+ */
+#define OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x25) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_VS    OP_3D_MEDIA(0x3, 0x0, 0x26) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_HS    OP_3D_MEDIA(0x3, 0x0, 0x27) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_DS    OP_3D_MEDIA(0x3, 0x0, 0x28) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_GS    OP_3D_MEDIA(0x3, 0x0, 0x29) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_PS    OP_3D_MEDIA(0x3, 0x0, 0x2A) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_VS    OP_3D_MEDIA(0x3, 0x0, 0x2B) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_HS    OP_3D_MEDIA(0x3, 0x0, 0x2C) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_DS    OP_3D_MEDIA(0x3, 0x0, 0x2D) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_GS    OP_3D_MEDIA(0x3, 0x0, 0x2E) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_PS    OP_3D_MEDIA(0x3, 0x0, 0x2F) /* IVB+ */
+#define OP_3DSTATE_URB_VS                       OP_3D_MEDIA(0x3, 0x0, 0x30) /* IVB+ */
+#define OP_3DSTATE_URB_HS                       OP_3D_MEDIA(0x3, 0x0, 0x31) /* IVB+ */
+#define OP_3DSTATE_URB_DS                       OP_3D_MEDIA(0x3, 0x0, 0x32) /* IVB+ */
+#define OP_3DSTATE_URB_GS                       OP_3D_MEDIA(0x3, 0x0, 0x33) /* IVB+ */
+#define OP_3DSTATE_GATHER_CONSTANT_VS           OP_3D_MEDIA(0x3, 0x0, 0x34) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_GS           OP_3D_MEDIA(0x3, 0x0, 0x35) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_HS           OP_3D_MEDIA(0x3, 0x0, 0x36) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_DS           OP_3D_MEDIA(0x3, 0x0, 0x37) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_PS           OP_3D_MEDIA(0x3, 0x0, 0x38) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTF_VS             OP_3D_MEDIA(0x3, 0x0, 0x39) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTF_PS             OP_3D_MEDIA(0x3, 0x0, 0x3A) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTI_VS             OP_3D_MEDIA(0x3, 0x0, 0x3B) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTI_PS             OP_3D_MEDIA(0x3, 0x0, 0x3C) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTB_VS             OP_3D_MEDIA(0x3, 0x0, 0x3D) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTB_PS             OP_3D_MEDIA(0x3, 0x0, 0x3E) /* HSW+ */
+#define OP_3DSTATE_DX9_LOCAL_VALID_VS           OP_3D_MEDIA(0x3, 0x0, 0x3F) /* HSW+ */
+#define OP_3DSTATE_DX9_LOCAL_VALID_PS           OP_3D_MEDIA(0x3, 0x0, 0x40) /* HSW+ */
+#define OP_3DSTATE_DX9_GENERATE_ACTIVE_VS       OP_3D_MEDIA(0x3, 0x0, 0x41) /* HSW+ */
+#define OP_3DSTATE_DX9_GENERATE_ACTIVE_PS       OP_3D_MEDIA(0x3, 0x0, 0x42) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_VS        OP_3D_MEDIA(0x3, 0x0, 0x43) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_GS        OP_3D_MEDIA(0x3, 0x0, 0x44) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_HS        OP_3D_MEDIA(0x3, 0x0, 0x45) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_DS        OP_3D_MEDIA(0x3, 0x0, 0x46) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_PS        OP_3D_MEDIA(0x3, 0x0, 0x47) /* HSW+ */
+
+#define OP_3DSTATE_VF_INSTANCING                OP_3D_MEDIA(0x3, 0x0, 0x49) /* BDW+ */
+#define OP_3DSTATE_VF_SGVS                      OP_3D_MEDIA(0x3, 0x0, 0x4A) /* BDW+ */
+#define OP_3DSTATE_VF_TOPOLOGY                  OP_3D_MEDIA(0x3, 0x0, 0x4B) /* BDW+ */
+#define OP_3DSTATE_WM_CHROMAKEY                 OP_3D_MEDIA(0x3, 0x0, 0x4C) /* BDW+ */
+#define OP_3DSTATE_PS_BLEND                     OP_3D_MEDIA(0x3, 0x0, 0x4D) /* BDW+ */
+#define OP_3DSTATE_WM_DEPTH_STENCIL             OP_3D_MEDIA(0x3, 0x0, 0x4E) /* BDW+ */
+#define OP_3DSTATE_PS_EXTRA                     OP_3D_MEDIA(0x3, 0x0, 0x4F) /* BDW+ */
+#define OP_3DSTATE_RASTER                       OP_3D_MEDIA(0x3, 0x0, 0x50) /* BDW+ */
+#define OP_3DSTATE_SBE_SWIZ                     OP_3D_MEDIA(0x3, 0x0, 0x51) /* BDW+ */
+#define OP_3DSTATE_WM_HZ_OP                     OP_3D_MEDIA(0x3, 0x0, 0x52) /* BDW+ */
+#define OP_3DSTATE_COMPONENT_PACKING            OP_3D_MEDIA(0x3, 0x0, 0x55) /* SKL+ */
+
+#define OP_3DSTATE_DRAWING_RECTANGLE            OP_3D_MEDIA(0x3, 0x1, 0x00)
+#define OP_3DSTATE_SAMPLER_PALETTE_LOAD0        OP_3D_MEDIA(0x3, 0x1, 0x02)
+#define OP_3DSTATE_CHROMA_KEY                   OP_3D_MEDIA(0x3, 0x1, 0x04)
+#define OP_SNB_3DSTATE_DEPTH_BUFFER             OP_3D_MEDIA(0x3, 0x1, 0x05)
+#define OP_3DSTATE_POLY_STIPPLE_OFFSET          OP_3D_MEDIA(0x3, 0x1, 0x06)
+#define OP_3DSTATE_POLY_STIPPLE_PATTERN         OP_3D_MEDIA(0x3, 0x1, 0x07)
+#define OP_3DSTATE_LINE_STIPPLE                 OP_3D_MEDIA(0x3, 0x1, 0x08)
+#define OP_3DSTATE_AA_LINE_PARAMS               OP_3D_MEDIA(0x3, 0x1, 0x0A)
+#define OP_3DSTATE_GS_SVB_INDEX                 OP_3D_MEDIA(0x3, 0x1, 0x0B)
+#define OP_3DSTATE_SAMPLER_PALETTE_LOAD1        OP_3D_MEDIA(0x3, 0x1, 0x0C)
+#define OP_3DSTATE_MULTISAMPLE_BDW              OP_3D_MEDIA(0x3, 0x0, 0x0D)
+#define OP_SNB_3DSTATE_STENCIL_BUFFER           OP_3D_MEDIA(0x3, 0x1, 0x0E)
+#define OP_SNB_3DSTATE_HIER_DEPTH_BUFFER        OP_3D_MEDIA(0x3, 0x1, 0x0F)
+#define OP_SNB_3DSTATE_CLEAR_PARAMS             OP_3D_MEDIA(0x3, 0x1, 0x10)
+#define OP_3DSTATE_MONOFILTER_SIZE              OP_3D_MEDIA(0x3, 0x1, 0x11)
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS       OP_3D_MEDIA(0x3, 0x1, 0x12) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS       OP_3D_MEDIA(0x3, 0x1, 0x13) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS       OP_3D_MEDIA(0x3, 0x1, 0x14) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS       OP_3D_MEDIA(0x3, 0x1, 0x15) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS       OP_3D_MEDIA(0x3, 0x1, 0x16) /* IVB+ */
+#define OP_3DSTATE_SO_DECL_LIST                 OP_3D_MEDIA(0x3, 0x1, 0x17)
+#define OP_3DSTATE_SO_BUFFER                    OP_3D_MEDIA(0x3, 0x1, 0x18)
+#define OP_3DSTATE_BINDING_TABLE_POOL_ALLOC     OP_3D_MEDIA(0x3, 0x1, 0x19) /* HSW+ */
+#define OP_3DSTATE_GATHER_POOL_ALLOC            OP_3D_MEDIA(0x3, 0x1, 0x1A) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1B) /* HSW+ */
+#define OP_3DSTATE_SAMPLE_PATTERN               OP_3D_MEDIA(0x3, 0x1, 0x1C)
+#define OP_PIPE_CONTROL                         OP_3D_MEDIA(0x3, 0x2, 0x00)
+#define OP_3DPRIMITIVE                          OP_3D_MEDIA(0x3, 0x3, 0x00)
+
+/* VCCP Command Parser */
+
+/*
+ * Below MFX and VBE cmd definition is from vaapi intel driver project (BSD License)
+ * git://anongit.freedesktop.org/vaapi/intel-driver
+ * src/i965_defines.h
+ *
+ */
+
+#define OP_MFX(pipeline, op, sub_opa, sub_opb)     \
+        (3 << 13 | \
+         (pipeline) << 11 | \
+         (op) << 8 | \
+         (sub_opa) << 5 | \
+         (sub_opb))
+
+#define OP_MFX_PIPE_MODE_SELECT                    OP_MFX(2, 0, 0, 0)  /* ALL */
+#define OP_MFX_SURFACE_STATE                       OP_MFX(2, 0, 0, 1)  /* ALL */
+#define OP_MFX_PIPE_BUF_ADDR_STATE                 OP_MFX(2, 0, 0, 2)  /* ALL */
+#define OP_MFX_IND_OBJ_BASE_ADDR_STATE             OP_MFX(2, 0, 0, 3)  /* ALL */
+#define OP_MFX_BSP_BUF_BASE_ADDR_STATE             OP_MFX(2, 0, 0, 4)  /* ALL */
+#define OP_2_0_0_5                                 OP_MFX(2, 0, 0, 5)  /* ALL */
+#define OP_MFX_STATE_POINTER                       OP_MFX(2, 0, 0, 6)  /* ALL */
+#define OP_MFX_QM_STATE                            OP_MFX(2, 0, 0, 7)  /* IVB+ */
+#define OP_MFX_FQM_STATE                           OP_MFX(2, 0, 0, 8)  /* IVB+ */
+#define OP_MFX_PAK_INSERT_OBJECT                   OP_MFX(2, 0, 2, 8)  /* IVB+ */
+#define OP_MFX_STITCH_OBJECT                       OP_MFX(2, 0, 2, 0xA)  /* IVB+ */
+
+#define OP_MFD_IT_OBJECT                           OP_MFX(2, 0, 1, 9) /* ALL */
+
+#define OP_MFX_WAIT                                OP_MFX(1, 0, 0, 0) /* IVB+ */
+#define OP_MFX_AVC_IMG_STATE                       OP_MFX(2, 1, 0, 0) /* ALL */
+#define OP_MFX_AVC_QM_STATE                        OP_MFX(2, 1, 0, 1) /* ALL */
+#define OP_MFX_AVC_DIRECTMODE_STATE                OP_MFX(2, 1, 0, 2) /* ALL */
+#define OP_MFX_AVC_SLICE_STATE                     OP_MFX(2, 1, 0, 3) /* ALL */
+#define OP_MFX_AVC_REF_IDX_STATE                   OP_MFX(2, 1, 0, 4) /* ALL */
+#define OP_MFX_AVC_WEIGHTOFFSET_STATE              OP_MFX(2, 1, 0, 5) /* ALL */
+#define OP_MFD_AVC_PICID_STATE                     OP_MFX(2, 1, 1, 5) /* HSW+ */
+#define OP_MFD_AVC_DPB_STATE                       OP_MFX(2, 1, 1, 6) /* IVB+ */
+#define OP_MFD_AVC_SLICEADDR                       OP_MFX(2, 1, 1, 7) /* IVB+ */
+#define OP_MFD_AVC_BSD_OBJECT                      OP_MFX(2, 1, 1, 8) /* ALL */
+#define OP_MFC_AVC_PAK_OBJECT                      OP_MFX(2, 1, 2, 9) /* ALL */
+
+#define OP_MFX_VC1_PRED_PIPE_STATE                 OP_MFX(2, 2, 0, 1) /* ALL */
+#define OP_MFX_VC1_DIRECTMODE_STATE                OP_MFX(2, 2, 0, 2) /* ALL */
+#define OP_MFD_VC1_SHORT_PIC_STATE                 OP_MFX(2, 2, 1, 0) /* IVB+ */
+#define OP_MFD_VC1_LONG_PIC_STATE                  OP_MFX(2, 2, 1, 1) /* IVB+ */
+#define OP_MFD_VC1_BSD_OBJECT                      OP_MFX(2, 2, 1, 8) /* ALL */
+
+#define OP_MFX_MPEG2_PIC_STATE                     OP_MFX(2, 3, 0, 0) /* ALL */
+#define OP_MFX_MPEG2_QM_STATE                      OP_MFX(2, 3, 0, 1) /* ALL */
+#define OP_MFD_MPEG2_BSD_OBJECT                    OP_MFX(2, 3, 1, 8) /* ALL */
+#define OP_MFC_MPEG2_SLICEGROUP_STATE              OP_MFX(2, 3, 2, 3) /* ALL */
+#define OP_MFC_MPEG2_PAK_OBJECT                    OP_MFX(2, 3, 2, 9) /* ALL */
+
+#define OP_MFX_2_6_0_0                             OP_MFX(2, 6, 0, 0) /* IVB+ */
+#define OP_MFX_2_6_0_8                             OP_MFX(2, 6, 0, 8) /* IVB+ */
+#define OP_MFX_2_6_0_9                             OP_MFX(2, 6, 0, 9) /* IVB+ */
+
+#define OP_MFX_JPEG_PIC_STATE                      OP_MFX(2, 7, 0, 0)
+#define OP_MFX_JPEG_HUFF_TABLE_STATE               OP_MFX(2, 7, 0, 2)
+#define OP_MFD_JPEG_BSD_OBJECT                     OP_MFX(2, 7, 1, 8)
+
+#define OP_VEB(pipeline, op, sub_opa, sub_opb) \
+        (3 << 13 | \
+         (pipeline) << 11 | \
+         (op) << 8 | \
+         (sub_opa) << 5 | \
+         (sub_opb))
+
+#define OP_VEB_SURFACE_STATE                       OP_VEB(2, 4, 0, 0)
+#define OP_VEB_STATE                               OP_VEB(2, 4, 0, 2)
+#define OP_VEB_DNDI_IECP_STATE                     OP_VEB(2, 4, 0, 3)
+
+struct parser_exec_state;
+
+typedef int (*parser_cmd_handler)(struct parser_exec_state *s);
+
+#define GVT_CMD_HASH_BITS   7
+
+/* which DWords need address fix */
+#define ADDR_FIX_1(x1)                  (1 << (x1))
+#define ADDR_FIX_2(x1, x2)              (ADDR_FIX_1(x1) | ADDR_FIX_1(x2))
+#define ADDR_FIX_3(x1, x2, x3)          (ADDR_FIX_1(x1) | ADDR_FIX_2(x2, x3))
+#define ADDR_FIX_4(x1, x2, x3, x4)      (ADDR_FIX_1(x1) | ADDR_FIX_3(x2, x3, x4))
+#define ADDR_FIX_5(x1, x2, x3, x4, x5)  (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5))
+
+struct cmd_info {
+        char *name;
+        u32 opcode;
+
+#define F_LEN_MASK      (1U<<0)
+#define F_LEN_CONST  1U
+#define F_LEN_VAR    0U
+
+/*
+ * command has its own ip advance logic
+ * e.g. MI_BATCH_START, MI_BATCH_END
+ */
+#define F_IP_ADVANCE_CUSTOM (1<<1)
+
+#define F_POST_HANDLE   (1<<2)
+        u32 flag;
+
+#define R_RCS   (1 << RCS)
+#define R_VCS1  (1 << VCS)
+#define R_VCS2  (1 << VCS2)
+#define R_VCS   (R_VCS1 | R_VCS2)
+#define R_BCS   (1 << BCS)
+#define R_VECS  (1 << VECS)
+#define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS)
+        /* rings that support this cmd: BLT/RCS/VCS/VECS */
+        uint16_t rings;
+
+        /* devices that support this cmd: SNB/IVB/HSW/... */
+        uint16_t devices;
+
+        /* which DWords are address that need fix up.
+         * bit 0 means a 32-bit non address operand in command
+         * bit 1 means address operand, which could be 32-bit
+         * or 64-bit depending on different architectures.(
+         * defined by "gmadr_bytes_in_cmd" in intel_gvt.
+         * No matter the address length, each address only takes
+         * one bit in the bitmap.
+         */
+        uint16_t addr_bitmap;
+
+        /* flag == F_LEN_CONST : command length
+         * flag == F_LEN_VAR : length bias bits
+         * Note: length is in DWord
+         */
+        uint8_t len;
+
+        parser_cmd_handler handler;
+};
+
+struct cmd_entry {
+        struct hlist_node hlist;
+        struct cmd_info *info;
+};
+
+enum {
+        RING_BUFFER_INSTRUCTION,
+        BATCH_BUFFER_INSTRUCTION,
+        BATCH_BUFFER_2ND_LEVEL,
+};
+
+enum {
+        GTT_BUFFER,
+        PPGTT_BUFFER
+};
+
+struct parser_exec_state {
+        struct intel_vgpu *vgpu;
+        int ring_id;
+
+        int buf_type;
+
+        /* batch buffer address type */
+        int buf_addr_type;
+
+        /* graphics memory address of ring buffer start */
+        unsigned long ring_start;
+        unsigned long ring_size;
+        unsigned long ring_head;
+        unsigned long ring_tail;
+
+        /* instruction graphics memory address */
+        unsigned long ip_gma;
+
+        /* mapped va of the instr_gma */
+        void *ip_va;
+        void *rb_va;
+
+        void *ret_bb_va;
+        /* next instruction when return from  batch buffer to ring buffer */
+        unsigned long ret_ip_gma_ring;
+
+        /* next instruction when return from 2nd batch buffer to batch buffer */
+        unsigned long ret_ip_gma_bb;
+
+        /* batch buffer address type (GTT or PPGTT)
+         * used when ret from 2nd level batch buffer
+         */
+        int saved_buf_addr_type;
+
+        struct cmd_info *info;
+
+        struct intel_vgpu_workload *workload;
+};
+
+#define gmadr_dw_number(s)      \
+        (s->vgpu->gvt->device_info.gmadr_bytes_in_cmd >> 2)
+
+static unsigned long bypass_scan_mask = 0;
+static bool bypass_batch_buffer_scan = true;
+
+/* ring ALL, type = 0 */
+static struct sub_op_bits sub_op_mi[] = {
+        {31, 29},
+        {28, 23},
+};
+
+static struct decode_info decode_info_mi = {
+        "MI",
+        OP_LEN_MI,
+        ARRAY_SIZE(sub_op_mi),
+        sub_op_mi,
+};
+
+/* ring RCS, command type 2 */
+static struct sub_op_bits sub_op_2d[] = {
+        {31, 29},
+        {28, 22},
+};
+
+static struct decode_info decode_info_2d = {
+        "2D",
+        OP_LEN_2D,
+        ARRAY_SIZE(sub_op_2d),
+        sub_op_2d,
+};
+
+/* ring RCS, command type 3 */
+static struct sub_op_bits sub_op_3d_media[] = {
+        {31, 29},
+        {28, 27},
+        {26, 24},
+        {23, 16},
+};
+
+static struct decode_info decode_info_3d_media = {
+        "3D_Media",
+        OP_LEN_3D_MEDIA,
+        ARRAY_SIZE(sub_op_3d_media),
+        sub_op_3d_media,
+};
+
+/* ring VCS, command type 3 */
+static struct sub_op_bits sub_op_mfx_vc[] = {
+        {31, 29},
+        {28, 27},
+        {26, 24},
+        {23, 21},
+        {20, 16},
+};
+
+static struct decode_info decode_info_mfx_vc = {
+        "MFX_VC",
+        OP_LEN_MFX_VC,
+        ARRAY_SIZE(sub_op_mfx_vc),
+        sub_op_mfx_vc,
+};
+
+/* ring VECS, command type 3 */
+static struct sub_op_bits sub_op_vebox[] = {
+        {31, 29},
+        {28, 27},
+        {26, 24},
+        {23, 21},
+        {20, 16},
+};
+
+static struct decode_info decode_info_vebox = {
+        "VEBOX",
+        OP_LEN_VEBOX,
+        ARRAY_SIZE(sub_op_vebox),
+        sub_op_vebox,
+};
+
+static struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
+        [RCS] = {
+                &decode_info_mi,
+                NULL,
+                NULL,
+                &decode_info_3d_media,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+        },
+
+        [VCS] = {
+                &decode_info_mi,
+                NULL,
+                NULL,
+                &decode_info_mfx_vc,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+        },
+
+        [BCS] = {
+                &decode_info_mi,
+                NULL,
+                &decode_info_2d,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+        },
+
+        [VECS] = {
+                &decode_info_mi,
+                NULL,
+                NULL,
+                &decode_info_vebox,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+        },
+
+        [VCS2] = {
+                &decode_info_mi,
+                NULL,
+                NULL,
+                &decode_info_mfx_vc,
+                NULL,
+                NULL,
+                NULL,
+                NULL,
+        },
+};
+
+static inline u32 get_opcode(u32 cmd, int ring_id)
+{
+        struct decode_info *d_info;
+
+        if (ring_id >= I915_NUM_ENGINES)
+                return INVALID_OP;
+
+        d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)];
+        if (d_info == NULL)
+                return INVALID_OP;
+
+        return cmd >> (32 - d_info->op_len);
+}
+
+static inline struct cmd_info *find_cmd_entry(struct intel_gvt *gvt,
+                unsigned int opcode, int ring_id)
+{
+        struct cmd_entry *e;
+
+        hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) {
+                if ((opcode == e->info->opcode) &&
+                                (e->info->rings & (1 << ring_id)))
+                        return e->info;
+        }
+        return NULL;
+}
+
+static inline struct cmd_info *get_cmd_info(struct intel_gvt *gvt,
+                u32 cmd, int ring_id)
+{
+        u32 opcode;
+
+        opcode = get_opcode(cmd, ring_id);
+        if (opcode == INVALID_OP)
+                return NULL;
+
+        return find_cmd_entry(gvt, opcode, ring_id);
+}
+
+static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low)
+{
+        return (cmd >> low) & ((1U << (hi - low + 1)) - 1);
+}
+
+static inline void print_opcode(u32 cmd, int ring_id)
+{
+        struct decode_info *d_info;
+        int i;
+
+        if (ring_id >= I915_NUM_ENGINES)
+                return;
+
+        d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)];
+        if (d_info == NULL)
+                return;
+
+        gvt_err("opcode=0x%x %s sub_ops:",
+                        cmd >> (32 - d_info->op_len), d_info->name);
+
+        for (i = 0; i < d_info->nr_sub_op; i++)
+                pr_err("0x%x ", sub_op_val(cmd, d_info->sub_op[i].hi,
+                                        d_info->sub_op[i].low));
+
+        pr_err("\n");
+}
+
+static inline u32 *cmd_ptr(struct parser_exec_state *s, int index)
+{
+        return s->ip_va + (index << 2);
+}
+
+static inline u32 cmd_val(struct parser_exec_state *s, int index)
+{
+        return *cmd_ptr(s, index);
+}
+
+static void parser_exec_state_dump(struct parser_exec_state *s)
+{
+        int cnt = 0;
+        int i;
+
+        gvt_err("  vgpu%d RING%d: ring_start(%08lx) ring_end(%08lx)"
+                        " ring_head(%08lx) ring_tail(%08lx)\n", s->vgpu->id,
+                        s->ring_id, s->ring_start, s->ring_start + s->ring_size,
+                        s->ring_head, s->ring_tail);
+
+        gvt_err("  %s %s ip_gma(%08lx) ",
+                        s->buf_type == RING_BUFFER_INSTRUCTION ?
+                        "RING_BUFFER" : "BATCH_BUFFER",
+                        s->buf_addr_type == GTT_BUFFER ?
+                        "GTT" : "PPGTT", s->ip_gma);
+
+        if (s->ip_va == NULL) {
+                gvt_err(" ip_va(NULL)");
+                return;
+        }
+
+        gvt_err("  ip_va=%p: %08x %08x %08x %08x\n",
+                        s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
+                        cmd_val(s, 2), cmd_val(s, 3));
+
+        print_opcode(cmd_val(s, 0), s->ring_id);
+
+        /* print the whole page to trace */
+        pr_err("    ip_va=%p: %08x %08x %08x %08x\n",
+                        s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
+                        cmd_val(s, 2), cmd_val(s, 3));
+
+        s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12);
+
+        while (cnt < 1024) {
+                pr_err("ip_va=%p: ", s->ip_va);
+                for (i = 0; i < 8; i++)
+                        pr_err("%08x ", cmd_val(s, i));
+                pr_err("\n");
+
+                s->ip_va += 8 * sizeof(u32);
+                cnt += 8;
+        }
+}
+
+static inline void update_ip_va(struct parser_exec_state *s)
+{
+        unsigned long len = 0;
+
+        if (WARN_ON(s->ring_head == s->ring_tail))
+                return;
+
+        if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+                unsigned long ring_top = s->ring_start + s->ring_size;
+
+                if (s->ring_head > s->ring_tail) {
+                        if (s->ip_gma >= s->ring_head && s->ip_gma < ring_top)
+                                len = (s->ip_gma - s->ring_head);
+                        else if (s->ip_gma >= s->ring_start &&
+                                        s->ip_gma <= s->ring_tail)
+                                len = (ring_top - s->ring_head) +
+                                        (s->ip_gma - s->ring_start);
+                } else
+                        len = (s->ip_gma - s->ring_head);
+
+                s->ip_va = s->rb_va + len;
+        } else {/* shadow batch buffer */
+                s->ip_va = s->ret_bb_va;
+        }
+}
+
+static inline int ip_gma_set(struct parser_exec_state *s,
+                unsigned long ip_gma)
+{
+        WARN_ON(!IS_ALIGNED(ip_gma, 4));
+
+        s->ip_gma = ip_gma;
+        update_ip_va(s);
+        return 0;
+}
+
+static inline int ip_gma_advance(struct parser_exec_state *s,
+                unsigned int dw_len)
+{
+        s->ip_gma += (dw_len << 2);
+
+        if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+                if (s->ip_gma >= s->ring_start + s->ring_size)
+                        s->ip_gma -= s->ring_size;
+                update_ip_va(s);
+        } else {
+                s->ip_va += (dw_len << 2);
+        }
+
+        return 0;
+}
+
+static inline int get_cmd_length(struct cmd_info *info, u32 cmd)
+{
+        if ((info->flag & F_LEN_MASK) == F_LEN_CONST)
+                return info->len;
+        else
+                return (cmd & ((1U << info->len) - 1)) + 2;
+        return 0;
+}
+
+static inline int cmd_length(struct parser_exec_state *s)
+{
+        return get_cmd_length(s->info, cmd_val(s, 0));
+}
+
+/* do not remove this, some platform may need clflush here */
+#define patch_value(s, addr, val) do { \
+        *addr = val; \
+} while (0)
+
+static bool is_shadowed_mmio(unsigned int offset)
+{
+        bool ret = false;
+
+        if ((offset == 0x2168) || /*BB current head register UDW */
+            (offset == 0x2140) || /*BB current header register */
+            (offset == 0x211c) || /*second BB header register UDW */
+            (offset == 0x2114)) { /*second BB header register UDW */
+                ret = true;
+        }
+        return ret;
+}
+
+static int cmd_reg_handler(struct parser_exec_state *s,
+        unsigned int offset, unsigned int index, char *cmd)
+{
+        struct intel_vgpu *vgpu = s->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+
+        if (offset + 4 > gvt->device_info.mmio_size) {
+                gvt_err("%s access to (%x) outside of MMIO range\n",
+                                cmd, offset);
+                return -EINVAL;
+        }
+
+        if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) {
+                gvt_err("vgpu%d: %s access to non-render register (%x)\n",
+                                s->vgpu->id, cmd, offset);
+                return 0;
+        }
+
+        if (is_shadowed_mmio(offset)) {
+                gvt_err("vgpu%d: found access of shadowed MMIO %x\n",
+                                s->vgpu->id, offset);
+                return 0;
+        }
+
+        if (offset == i915_mmio_reg_offset(DERRMR) ||
+                offset == i915_mmio_reg_offset(FORCEWAKE_MT)) {
+                /* Writing to HW VGT_PVINFO_PAGE offset will be discarded */
+                patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE);
+        }
+
+        /* TODO: Update the global mask if this MMIO is a masked-MMIO */
+        intel_gvt_mmio_set_cmd_accessed(gvt, offset);
+        return 0;
+}
+
+#define cmd_reg(s, i) \
+        (cmd_val(s, i) & GENMASK(22, 2))
+
+#define cmd_reg_inhibit(s, i) \
+        (cmd_val(s, i) & GENMASK(22, 18))
+
+#define cmd_gma(s, i) \
+        (cmd_val(s, i) & GENMASK(31, 2))
+
+#define cmd_gma_hi(s, i) \
+        (cmd_val(s, i) & GENMASK(15, 0))
+
+static int cmd_handler_lri(struct parser_exec_state *s)
+{
+        int i, ret = 0;
+        int cmd_len = cmd_length(s);
+        struct intel_gvt *gvt = s->vgpu->gvt;
+
+        for (i = 1; i < cmd_len; i += 2) {
+                if (IS_BROADWELL(gvt->dev_priv) &&
+                                (s->ring_id != RCS)) {
+                        if (s->ring_id == BCS &&
+                                        cmd_reg(s, i) ==
+                                        i915_mmio_reg_offset(DERRMR))
+                                ret |= 0;
+                        else
+                                ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+                }
+                if (ret)
+                        break;
+                ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri");
+        }
+        return ret;
+}
+
+static int cmd_handler_lrr(struct parser_exec_state *s)
+{
+        int i, ret = 0;
+        int cmd_len = cmd_length(s);
+
+        for (i = 1; i < cmd_len; i += 2) {
+                if (IS_BROADWELL(s->vgpu->gvt->dev_priv))
+                        ret |= ((cmd_reg_inhibit(s, i) ||
+                                        (cmd_reg_inhibit(s, i + 1)))) ?
+                                -EINVAL : 0;
+                if (ret)
+                        break;
+                ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src");
+                ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst");
+        }
+        return ret;
+}
+
+static inline int cmd_address_audit(struct parser_exec_state *s,
+                unsigned long guest_gma, int op_size, bool index_mode);
+
+static int cmd_handler_lrm(struct parser_exec_state *s)
+{
+        struct intel_gvt *gvt = s->vgpu->gvt;
+        int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
+        unsigned long gma;
+        int i, ret = 0;
+        int cmd_len = cmd_length(s);
+
+        for (i = 1; i < cmd_len;) {
+                if (IS_BROADWELL(gvt->dev_priv))
+                        ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+                if (ret)
+                        break;
+                ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm");
+                if (cmd_val(s, 0) & (1 << 22)) {
+                        gma = cmd_gma(s, i + 1);
+                        if (gmadr_bytes == 8)
+                                gma |= (cmd_gma_hi(s, i + 2)) << 32;
+                        ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+                }
+                i += gmadr_dw_number(s) + 1;
+        }
+        return ret;
+}
+
+static int cmd_handler_srm(struct parser_exec_state *s)
+{
+        int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+        unsigned long gma;
+        int i, ret = 0;
+        int cmd_len = cmd_length(s);
+
+        for (i = 1; i < cmd_len;) {
+                ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm");
+                if (cmd_val(s, 0) & (1 << 22)) {
+                        gma = cmd_gma(s, i + 1);
+                        if (gmadr_bytes == 8)
+                                gma |= (cmd_gma_hi(s, i + 2)) << 32;
+                        ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+                }
+                i += gmadr_dw_number(s) + 1;
+        }
+        return ret;
+}
+
+struct cmd_interrupt_event {
+        int pipe_control_notify;
+        int mi_flush_dw;
+        int mi_user_interrupt;
+};
+
+static struct cmd_interrupt_event cmd_interrupt_events[] = {
+        [RCS] = {
+                .pipe_control_notify = RCS_PIPE_CONTROL,
+                .mi_flush_dw = INTEL_GVT_EVENT_RESERVED,
+                .mi_user_interrupt = RCS_MI_USER_INTERRUPT,
+        },
+        [BCS] = {
+                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+                .mi_flush_dw = BCS_MI_FLUSH_DW,
+                .mi_user_interrupt = BCS_MI_USER_INTERRUPT,
+        },
+        [VCS] = {
+                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+                .mi_flush_dw = VCS_MI_FLUSH_DW,
+                .mi_user_interrupt = VCS_MI_USER_INTERRUPT,
+        },
+        [VCS2] = {
+                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+                .mi_flush_dw = VCS2_MI_FLUSH_DW,
+                .mi_user_interrupt = VCS2_MI_USER_INTERRUPT,
+        },
+        [VECS] = {
+                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+                .mi_flush_dw = VECS_MI_FLUSH_DW,
+                .mi_user_interrupt = VECS_MI_USER_INTERRUPT,
+        },
+};
+
+static int cmd_handler_pipe_control(struct parser_exec_state *s)
+{
+        int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+        unsigned long gma;
+        bool index_mode = false;
+        unsigned int post_sync;
+        int ret = 0;
+
+        post_sync = (cmd_val(s, 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14;
+
+        /* LRI post sync */
+        if (cmd_val(s, 1) & PIPE_CONTROL_MMIO_WRITE)
+                ret = cmd_reg_handler(s, cmd_reg(s, 2), 1, "pipe_ctrl");
+        /* post sync */
+        else if (post_sync) {
+                if (post_sync == 2)
+                        ret = cmd_reg_handler(s, 0x2350, 1, "pipe_ctrl");
+                else if (post_sync == 3)
+                        ret = cmd_reg_handler(s, 0x2358, 1, "pipe_ctrl");
+                else if (post_sync == 1) {
+                        /* check ggtt*/
+                        if ((cmd_val(s, 2) & (1 << 2))) {
+                                gma = cmd_val(s, 2) & GENMASK(31, 3);
+                                if (gmadr_bytes == 8)
+                                        gma |= (cmd_gma_hi(s, 3)) << 32;
+                                /* Store Data Index */
+                                if (cmd_val(s, 1) & (1 << 21))
+                                        index_mode = true;
+                                ret |= cmd_address_audit(s, gma, sizeof(u64),
+                                                index_mode);
+                        }
+                }
+        }
+
+        if (ret)
+                return ret;
+
+        if (cmd_val(s, 1) & PIPE_CONTROL_NOTIFY)
+                set_bit(cmd_interrupt_events[s->ring_id].pipe_control_notify,
+                                s->workload->pending_events);
+        return 0;
+}
+
+static int cmd_handler_mi_user_interrupt(struct parser_exec_state *s)
+{
+        set_bit(cmd_interrupt_events[s->ring_id].mi_user_interrupt,
+                        s->workload->pending_events);
+        return 0;
+}
+
+static int cmd_advance_default(struct parser_exec_state *s)
+{
+        return ip_gma_advance(s, cmd_length(s));
+}
+
+static int cmd_handler_mi_batch_buffer_end(struct parser_exec_state *s)
+{
+        int ret;
+
+        if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
+                s->buf_type = BATCH_BUFFER_INSTRUCTION;
+                ret = ip_gma_set(s, s->ret_ip_gma_bb);
+                s->buf_addr_type = s->saved_buf_addr_type;
+        } else {
+                s->buf_type = RING_BUFFER_INSTRUCTION;
+                s->buf_addr_type = GTT_BUFFER;
+                if (s->ret_ip_gma_ring >= s->ring_start + s->ring_size)
+                        s->ret_ip_gma_ring -= s->ring_size;
+                ret = ip_gma_set(s, s->ret_ip_gma_ring);
+        }
+        return ret;
+}
+
+struct mi_display_flip_command_info {
+        int pipe;
+        int plane;
+        int event;
+        i915_reg_t stride_reg;
+        i915_reg_t ctrl_reg;
+        i915_reg_t surf_reg;
+        u64 stride_val;
+        u64 tile_val;
+        u64 surf_val;
+        bool async_flip;
+};
+
+struct plane_code_mapping {
+        int pipe;
+        int plane;
+        int event;
+};
+
+static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
+                struct mi_display_flip_command_info *info)
+{
+        struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+        struct plane_code_mapping gen8_plane_code[] = {
+                [0] = {PIPE_A, PLANE_A, PRIMARY_A_FLIP_DONE},
+                [1] = {PIPE_B, PLANE_A, PRIMARY_B_FLIP_DONE},
+                [2] = {PIPE_A, PLANE_B, SPRITE_A_FLIP_DONE},
+                [3] = {PIPE_B, PLANE_B, SPRITE_B_FLIP_DONE},
+                [4] = {PIPE_C, PLANE_A, PRIMARY_C_FLIP_DONE},
+                [5] = {PIPE_C, PLANE_B, SPRITE_C_FLIP_DONE},
+        };
+        u32 dword0, dword1, dword2;
+        u32 v;
+
+        dword0 = cmd_val(s, 0);
+        dword1 = cmd_val(s, 1);
+        dword2 = cmd_val(s, 2);
+
+        v = (dword0 & GENMASK(21, 19)) >> 19;
+        if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code)))
+                return -EINVAL;
+
+        info->pipe = gen8_plane_code[v].pipe;
+        info->plane = gen8_plane_code[v].plane;
+        info->event = gen8_plane_code[v].event;
+        info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
+        info->tile_val = (dword1 & 0x1);
+        info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
+        info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
+
+        if (info->plane == PLANE_A) {
+                info->ctrl_reg = DSPCNTR(info->pipe);
+                info->stride_reg = DSPSTRIDE(info->pipe);
+                info->surf_reg = DSPSURF(info->pipe);
+        } else if (info->plane == PLANE_B) {
+                info->ctrl_reg = SPRCTL(info->pipe);
+                info->stride_reg = SPRSTRIDE(info->pipe);
+                info->surf_reg = SPRSURF(info->pipe);
+        } else {
+                WARN_ON(1);
+                return -EINVAL;
+        }
+        return 0;
+}
+
+static int skl_decode_mi_display_flip(struct parser_exec_state *s,
+                struct mi_display_flip_command_info *info)
+{
+        struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+        u32 dword0 = cmd_val(s, 0);
+        u32 dword1 = cmd_val(s, 1);
+        u32 dword2 = cmd_val(s, 2);
+        u32 plane = (dword0 & GENMASK(12, 8)) >> 8;
+
+        switch (plane) {
+        case MI_DISPLAY_FLIP_SKL_PLANE_1_A:
+                info->pipe = PIPE_A;
+                info->event = PRIMARY_A_FLIP_DONE;
+                break;
+        case MI_DISPLAY_FLIP_SKL_PLANE_1_B:
+                info->pipe = PIPE_B;
+                info->event = PRIMARY_B_FLIP_DONE;
+                break;
+        case MI_DISPLAY_FLIP_SKL_PLANE_1_C:
+                info->pipe = PIPE_B;
+                info->event = PRIMARY_C_FLIP_DONE;
+                break;
+        default:
+                gvt_err("unknown plane code %d\n", plane);
+                return -EINVAL;
+        }
+
+        info->pipe = PRIMARY_PLANE;
+        info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
+        info->tile_val = (dword1 & GENMASK(2, 0));
+        info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
+        info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
+
+        info->ctrl_reg = DSPCNTR(info->pipe);
+        info->stride_reg = DSPSTRIDE(info->pipe);
+        info->surf_reg = DSPSURF(info->pipe);
+
+        return 0;
+}
+
+static int gen8_check_mi_display_flip(struct parser_exec_state *s,
+                struct mi_display_flip_command_info *info)
+{
+        struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+        u32 stride, tile;
+
+        if (!info->async_flip)
+                return 0;
+
+        if (IS_SKYLAKE(dev_priv)) {
+                stride = vgpu_vreg(s->vgpu, info->stride_reg) & GENMASK(9, 0);
+                tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) &
+                                GENMASK(12, 10)) >> 10;
+        } else {
+                stride = (vgpu_vreg(s->vgpu, info->stride_reg) &
+                                GENMASK(15, 6)) >> 6;
+                tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10;
+        }
+
+        if (stride != info->stride_val)
+                gvt_dbg_cmd("cannot change stride during async flip\n");
+
+        if (tile != info->tile_val)
+                gvt_dbg_cmd("cannot change tile during async flip\n");
+
+        return 0;
+}
+
+static int gen8_update_plane_mmio_from_mi_display_flip(
+                struct parser_exec_state *s,
+                struct mi_display_flip_command_info *info)
+{
+        struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+        struct intel_vgpu *vgpu = s->vgpu;
+
+#define write_bits(reg, e, s, v) do { \
+        vgpu_vreg(vgpu, reg) &= ~GENMASK(e, s); \
+        vgpu_vreg(vgpu, reg) |= (v << s); \
+} while (0)
+
+        write_bits(info->surf_reg, 31, 12, info->surf_val);
+        if (IS_SKYLAKE(dev_priv))
+                write_bits(info->stride_reg, 9, 0, info->stride_val);
+        else
+                write_bits(info->stride_reg, 15, 6, info->stride_val);
+        write_bits(info->ctrl_reg, IS_SKYLAKE(dev_priv) ? 12 : 10,
+                   10, info->tile_val);
+
+#undef write_bits
+
+        vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++;
+        intel_vgpu_trigger_virtual_event(vgpu, info->event);
+        return 0;
+}
+
+static int decode_mi_display_flip(struct parser_exec_state *s,
+                struct mi_display_flip_command_info *info)
+{
+        struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+
+        if (IS_BROADWELL(dev_priv))
+                return gen8_decode_mi_display_flip(s, info);
+        if (IS_SKYLAKE(dev_priv))
+                return skl_decode_mi_display_flip(s, info);
+
+        return -ENODEV;
+}
+
+static int check_mi_display_flip(struct parser_exec_state *s,
+                struct mi_display_flip_command_info *info)
+{
+        struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+
+        if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))
+                return gen8_check_mi_display_flip(s, info);
+        return -ENODEV;
+}
+
+static int update_plane_mmio_from_mi_display_flip(
+                struct parser_exec_state *s,
+                struct mi_display_flip_command_info *info)
+{
+        struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+
+        if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))
+                return gen8_update_plane_mmio_from_mi_display_flip(s, info);
+        return -ENODEV;
+}
+
+static int cmd_handler_mi_display_flip(struct parser_exec_state *s)
+{
+        struct mi_display_flip_command_info info;
+        int ret;
+        int i;
+        int len = cmd_length(s);
+
+        ret = decode_mi_display_flip(s, &info);
+        if (ret) {
+                gvt_err("fail to decode MI display flip command\n");
+                return ret;
+        }
+
+        ret = check_mi_display_flip(s, &info);
+        if (ret) {
+                gvt_err("invalid MI display flip command\n");
+                return ret;
+        }
+
+        ret = update_plane_mmio_from_mi_display_flip(s, &info);
+        if (ret) {
+                gvt_err("fail to update plane mmio\n");
+                return ret;
+        }
+
+        for (i = 0; i < len; i++)
+                patch_value(s, cmd_ptr(s, i), MI_NOOP);
+        return 0;
+}
+
+static bool is_wait_for_flip_pending(u32 cmd)
+{
+        return cmd & (MI_WAIT_FOR_PLANE_A_FLIP_PENDING |
+                        MI_WAIT_FOR_PLANE_B_FLIP_PENDING |
+                        MI_WAIT_FOR_PLANE_C_FLIP_PENDING |
+                        MI_WAIT_FOR_SPRITE_A_FLIP_PENDING |
+                        MI_WAIT_FOR_SPRITE_B_FLIP_PENDING |
+                        MI_WAIT_FOR_SPRITE_C_FLIP_PENDING);
+}
+
+static int cmd_handler_mi_wait_for_event(struct parser_exec_state *s)
+{
+        u32 cmd = cmd_val(s, 0);
+
+        if (!is_wait_for_flip_pending(cmd))
+                return 0;
+
+        patch_value(s, cmd_ptr(s, 0), MI_NOOP);
+        return 0;
+}
+
+static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index)
+{
+        unsigned long addr;
+        unsigned long gma_high, gma_low;
+        int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+
+        if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8))
+                return INTEL_GVT_INVALID_ADDR;
+
+        gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK;
+        if (gmadr_bytes == 4) {
+                addr = gma_low;
+        } else {
+                gma_high = cmd_val(s, index + 1) & BATCH_BUFFER_ADDR_HIGH_MASK;
+                addr = (((unsigned long)gma_high) << 32) | gma_low;
+        }
+        return addr;
+}
+
+static inline int cmd_address_audit(struct parser_exec_state *s,
+                unsigned long guest_gma, int op_size, bool index_mode)
+{
+        struct intel_vgpu *vgpu = s->vgpu;
+        u32 max_surface_size = vgpu->gvt->device_info.max_surface_size;
+        int i;
+        int ret;
+
+        if (op_size > max_surface_size) {
+                gvt_err("command address audit fail name %s\n", s->info->name);
+                return -EINVAL;
+        }
+
+        if (index_mode) {
+                if (guest_gma >= GTT_PAGE_SIZE / sizeof(u64)) {
+                        ret = -EINVAL;
+                        goto err;
+                }
+        } else if ((!vgpu_gmadr_is_valid(s->vgpu, guest_gma)) ||
+                        (!vgpu_gmadr_is_valid(s->vgpu,
+                                              guest_gma + op_size - 1))) {
+                ret = -EINVAL;
+                goto err;
+        }
+        return 0;
+err:
+        gvt_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n",
+                        s->info->name, guest_gma, op_size);
+
+        pr_err("cmd dump: ");
+        for (i = 0; i < cmd_length(s); i++) {
+                if (!(i % 4))
+                        pr_err("\n%08x ", cmd_val(s, i));
+                else
+                        pr_err("%08x ", cmd_val(s, i));
+        }
+        pr_err("\nvgpu%d: aperture 0x%llx - 0x%llx, hidden 0x%llx - 0x%llx\n",
+                        vgpu->id,
+                        vgpu_aperture_gmadr_base(vgpu),
+                        vgpu_aperture_gmadr_end(vgpu),
+                        vgpu_hidden_gmadr_base(vgpu),
+                        vgpu_hidden_gmadr_end(vgpu));
+        return ret;
+}
+
+static int cmd_handler_mi_store_data_imm(struct parser_exec_state *s)
+{
+        int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+        int op_size = (cmd_length(s) - 3) * sizeof(u32);
+        int core_id = (cmd_val(s, 2) & (1 << 0)) ? 1 : 0;
+        unsigned long gma, gma_low, gma_high;
+        int ret = 0;
+
+        /* check ppggt */
+        if (!(cmd_val(s, 0) & (1 << 22)))
+                return 0;
+
+        gma = cmd_val(s, 2) & GENMASK(31, 2);
+
+        if (gmadr_bytes == 8) {
+                gma_low = cmd_val(s, 1) & GENMASK(31, 2);
+                gma_high = cmd_val(s, 2) & GENMASK(15, 0);
+                gma = (gma_high << 32) | gma_low;
+                core_id = (cmd_val(s, 1) & (1 << 0)) ? 1 : 0;
+        }
+        ret = cmd_address_audit(s, gma + op_size * core_id, op_size, false);
+        return ret;
+}
+
+static inline int unexpected_cmd(struct parser_exec_state *s)
+{
+        gvt_err("vgpu%d: Unexpected %s in command buffer!\n",
+                        s->vgpu->id, s->info->name);
+        return -EINVAL;
+}
+
+static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s)
+{
+        return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_report_perf_count(struct parser_exec_state *s)
+{
+        return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_op_2e(struct parser_exec_state *s)
+{
+        return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_op_2f(struct parser_exec_state *s)
+{
+        int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+        int op_size = ((1 << (cmd_val(s, 0) & GENMASK(20, 19) >> 19)) *
+                        sizeof(u32));
+        unsigned long gma, gma_high;
+        int ret = 0;
+
+        if (!(cmd_val(s, 0) & (1 << 22)))
+                return ret;
+
+        gma = cmd_val(s, 1) & GENMASK(31, 2);
+        if (gmadr_bytes == 8) {
+                gma_high = cmd_val(s, 2) & GENMASK(15, 0);
+                gma = (gma_high << 32) | gma;
+        }
+        ret = cmd_address_audit(s, gma, op_size, false);
+        return ret;
+}
+
+static int cmd_handler_mi_store_data_index(struct parser_exec_state *s)
+{
+        return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_clflush(struct parser_exec_state *s)
+{
+        return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_conditional_batch_buffer_end(
+                struct parser_exec_state *s)
+{
+        return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_update_gtt(struct parser_exec_state *s)
+{
+        return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_flush_dw(struct parser_exec_state *s)
+{
+        int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+        unsigned long gma;
+        bool index_mode = false;
+        int ret = 0;
+
+        /* Check post-sync and ppgtt bit */
+        if (((cmd_val(s, 0) >> 14) & 0x3) && (cmd_val(s, 1) & (1 << 2))) {
+                gma = cmd_val(s, 1) & GENMASK(31, 3);
+                if (gmadr_bytes == 8)
+                        gma |= (cmd_val(s, 2) & GENMASK(15, 0)) << 32;
+                /* Store Data Index */
+                if (cmd_val(s, 0) & (1 << 21))
+                        index_mode = true;
+                ret = cmd_address_audit(s, gma, sizeof(u64), index_mode);
+        }
+        /* Check notify bit */
+        if ((cmd_val(s, 0) & (1 << 8)))
+                set_bit(cmd_interrupt_events[s->ring_id].mi_flush_dw,
+                                s->workload->pending_events);
+        return ret;
+}
+
+static void addr_type_update_snb(struct parser_exec_state *s)
+{
+        if ((s->buf_type == RING_BUFFER_INSTRUCTION) &&
+                        (BATCH_BUFFER_ADR_SPACE_BIT(cmd_val(s, 0)) == 1)) {
+                s->buf_addr_type = PPGTT_BUFFER;
+        }
+}
+
+
+static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm,
+                unsigned long gma, unsigned long end_gma, void *va)
+{
+        unsigned long copy_len, offset;
+        unsigned long len = 0;
+        unsigned long gpa;
+
+        while (gma != end_gma) {
+                gpa = intel_vgpu_gma_to_gpa(mm, gma);
+                if (gpa == INTEL_GVT_INVALID_ADDR) {
+                        gvt_err("invalid gma address: %lx\n", gma);
+                        return -EFAULT;
+                }
+
+                offset = gma & (GTT_PAGE_SIZE - 1);
+
+                copy_len = (end_gma - gma) >= (GTT_PAGE_SIZE - offset) ?
+                        GTT_PAGE_SIZE - offset : end_gma - gma;
+
+                intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len);
+
+                len += copy_len;
+                gma += copy_len;
+        }
+        return 0;
+}
+
+
+/*
+ * Check whether a batch buffer needs to be scanned. Currently
+ * the only criteria is based on privilege.
+ */
+static int batch_buffer_needs_scan(struct parser_exec_state *s)
+{
+        struct intel_gvt *gvt = s->vgpu->gvt;
+
+        if (bypass_batch_buffer_scan)
+                return 0;
+
+        if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+                /* BDW decides privilege based on address space */
+                if (cmd_val(s, 0) & (1 << 8))
+                        return 0;
+        }
+        return 1;
+}
+
+static uint32_t find_bb_size(struct parser_exec_state *s)
+{
+        unsigned long gma = 0;
+        struct cmd_info *info;
+        uint32_t bb_size = 0;
+        uint32_t cmd_len = 0;
+        bool met_bb_end = false;
+        u32 cmd;
+
+        /* get the start gm address of the batch buffer */
+        gma = get_gma_bb_from_cmd(s, 1);
+        cmd = cmd_val(s, 0);
+
+        info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
+        if (info == NULL) {
+                gvt_err("unknown cmd 0x%x, opcode=0x%x\n",
+                                cmd, get_opcode(cmd, s->ring_id));
+                return -EINVAL;
+        }
+        do {
+                copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+                                gma, gma + 4, &cmd);
+                info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
+                if (info == NULL) {
+                        gvt_err("unknown cmd 0x%x, opcode=0x%x\n",
+                                cmd, get_opcode(cmd, s->ring_id));
+                        return -EINVAL;
+                }
+
+                if (info->opcode == OP_MI_BATCH_BUFFER_END) {
+                        met_bb_end = true;
+                } else if (info->opcode == OP_MI_BATCH_BUFFER_START) {
+                        if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) {
+                                /* chained batch buffer */
+                                met_bb_end = true;
+                        }
+                }
+                cmd_len = get_cmd_length(info, cmd) << 2;
+                bb_size += cmd_len;
+                gma += cmd_len;
+
+        } while (!met_bb_end);
+
+        return bb_size;
+}
+
+static int perform_bb_shadow(struct parser_exec_state *s)
+{
+        struct intel_shadow_bb_entry *entry_obj;
+        unsigned long gma = 0;
+        uint32_t bb_size;
+        void *dst = NULL;
+        int ret = 0;
+
+        /* get the start gm address of the batch buffer */
+        gma = get_gma_bb_from_cmd(s, 1);
+
+        /* get the size of the batch buffer */
+        bb_size = find_bb_size(s);
+
+        /* allocate shadow batch buffer */
+        entry_obj = kmalloc(sizeof(*entry_obj), GFP_KERNEL);
+        if (entry_obj == NULL)
+                return -ENOMEM;
+
+        entry_obj->obj =
+                i915_gem_object_create(&(s->vgpu->gvt->dev_priv->drm),
+                                       roundup(bb_size, PAGE_SIZE));
+        if (IS_ERR(entry_obj->obj)) {
+                ret = PTR_ERR(entry_obj->obj);
+                goto free_entry;
+        }
+        entry_obj->len = bb_size;
+        INIT_LIST_HEAD(&entry_obj->list);
+
+        dst = i915_gem_object_pin_map(entry_obj->obj, I915_MAP_WB);
+        if (IS_ERR(dst)) {
+                ret = PTR_ERR(dst);
+                goto put_obj;
+        }
+
+        ret = i915_gem_object_set_to_cpu_domain(entry_obj->obj, false);
+        if (ret) {
+                gvt_err("failed to set shadow batch to CPU\n");
+                goto unmap_src;
+        }
+
+        entry_obj->va = dst;
+        entry_obj->bb_start_cmd_va = s->ip_va;
+
+        /* copy batch buffer to shadow batch buffer*/
+        ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+                              gma, gma + bb_size,
+                              dst);
+        if (ret) {
+                gvt_err("fail to copy guest ring buffer\n");
+                goto unmap_src;
+        }
+
+        list_add(&entry_obj->list, &s->workload->shadow_bb);
+        /*
+         * ip_va saves the virtual address of the shadow batch buffer, while
+         * ip_gma saves the graphics address of the original batch buffer.
+         * As the shadow batch buffer is just a copy from the originial one,
+         * it should be right to use shadow batch buffer'va and original batch
+         * buffer's gma in pair. After all, we don't want to pin the shadow
+         * buffer here (too early).
+         */
+        s->ip_va = dst;
+        s->ip_gma = gma;
+
+        return 0;
+
+unmap_src:
+        i915_gem_object_unpin_map(entry_obj->obj);
+put_obj:
+        i915_gem_object_put(entry_obj->obj);
+free_entry:
+        kfree(entry_obj);
+        return ret;
+}
+
+static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
+{
+        bool second_level;
+        int ret = 0;
+
+        if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
+                gvt_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n");
+                return -EINVAL;
+        }
+
+        second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1;
+        if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) {
+                gvt_err("Jumping to 2nd level BB from RB is not allowed\n");
+                return -EINVAL;
+        }
+
+        s->saved_buf_addr_type = s->buf_addr_type;
+        addr_type_update_snb(s);
+        if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+                s->ret_ip_gma_ring = s->ip_gma + cmd_length(s) * sizeof(u32);
+                s->buf_type = BATCH_BUFFER_INSTRUCTION;
+        } else if (second_level) {
+                s->buf_type = BATCH_BUFFER_2ND_LEVEL;
+                s->ret_ip_gma_bb = s->ip_gma + cmd_length(s) * sizeof(u32);
+                s->ret_bb_va = s->ip_va + cmd_length(s) * sizeof(u32);
+        }
+
+        if (batch_buffer_needs_scan(s)) {
+                ret = perform_bb_shadow(s);
+                if (ret < 0)
+                        gvt_err("invalid shadow batch buffer\n");
+        } else {
+                /* emulate a batch buffer end to do return right */
+                ret = cmd_handler_mi_batch_buffer_end(s);
+                if (ret < 0)
+                        return ret;
+        }
+
+        return ret;
+}
+
+static struct cmd_info cmd_info[] = {
+        {"MI_NOOP", OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
+
+        {"MI_SET_PREDICATE", OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL,
+                0, 1, NULL},
+
+        {"MI_USER_INTERRUPT", OP_MI_USER_INTERRUPT, F_LEN_CONST, R_ALL, D_ALL,
+                0, 1, cmd_handler_mi_user_interrupt},
+
+        {"MI_WAIT_FOR_EVENT", OP_MI_WAIT_FOR_EVENT, F_LEN_CONST, R_RCS | R_BCS,
+                D_ALL, 0, 1, cmd_handler_mi_wait_for_event},
+
+        {"MI_FLUSH", OP_MI_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
+
+        {"MI_ARB_CHECK", OP_MI_ARB_CHECK, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+                NULL},
+
+        {"MI_RS_CONTROL", OP_MI_RS_CONTROL, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
+                NULL},
+
+        {"MI_REPORT_HEAD", OP_MI_REPORT_HEAD, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+                NULL},
+
+        {"MI_ARB_ON_OFF", OP_MI_ARB_ON_OFF, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+                NULL},
+
+        {"MI_URB_ATOMIC_ALLOC", OP_MI_URB_ATOMIC_ALLOC, F_LEN_CONST, R_RCS,
+                D_ALL, 0, 1, NULL},
+
+        {"MI_BATCH_BUFFER_END", OP_MI_BATCH_BUFFER_END,
+                F_IP_ADVANCE_CUSTOM | F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+                cmd_handler_mi_batch_buffer_end},
+
+        {"MI_SUSPEND_FLUSH", OP_MI_SUSPEND_FLUSH, F_LEN_CONST, R_ALL, D_ALL,
+                0, 1, NULL},
+
+        {"MI_PREDICATE", OP_MI_PREDICATE, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
+                NULL},
+
+        {"MI_TOPOLOGY_FILTER", OP_MI_TOPOLOGY_FILTER, F_LEN_CONST, R_ALL,
+                D_ALL, 0, 1, NULL},
+
+        {"MI_SET_APPID", OP_MI_SET_APPID, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+                NULL},
+
+        {"MI_RS_CONTEXT", OP_MI_RS_CONTEXT, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
+                NULL},
+
+        {"MI_DISPLAY_FLIP", OP_MI_DISPLAY_FLIP, F_LEN_VAR | F_POST_HANDLE,
+                R_RCS | R_BCS, D_ALL, 0, 8, cmd_handler_mi_display_flip},
+
+        {"MI_SEMAPHORE_MBOX", OP_MI_SEMAPHORE_MBOX, F_LEN_VAR, R_ALL, D_ALL,
+                0, 8, NULL},
+
+        {"MI_MATH", OP_MI_MATH, F_LEN_VAR, R_ALL, D_ALL, 0, 8, NULL},
+
+        {"MI_URB_CLEAR", OP_MI_URB_CLEAR, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"ME_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL, F_LEN_VAR, R_ALL,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"ME_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT, F_LEN_VAR, R_ALL, D_BDW_PLUS,
+                ADDR_FIX_1(2), 8, cmd_handler_mi_semaphore_wait},
+
+        {"MI_STORE_DATA_IMM", OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS,
+                ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm},
+
+        {"MI_STORE_DATA_INDEX", OP_MI_STORE_DATA_INDEX, F_LEN_VAR, R_ALL, D_ALL,
+                0, 8, cmd_handler_mi_store_data_index},
+
+        {"MI_LOAD_REGISTER_IMM", OP_MI_LOAD_REGISTER_IMM, F_LEN_VAR, R_ALL,
+                D_ALL, 0, 8, cmd_handler_lri},
+
+        {"MI_UPDATE_GTT", OP_MI_UPDATE_GTT, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 10,
+                cmd_handler_mi_update_gtt},
+
+        {"MI_STORE_REGISTER_MEM", OP_MI_STORE_REGISTER_MEM, F_LEN_VAR, R_ALL,
+                D_ALL, ADDR_FIX_1(2), 8, cmd_handler_srm},
+
+        {"MI_FLUSH_DW", OP_MI_FLUSH_DW, F_LEN_VAR, R_ALL, D_ALL, 0, 6,
+                cmd_handler_mi_flush_dw},
+
+        {"MI_CLFLUSH", OP_MI_CLFLUSH, F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(1),
+                10, cmd_handler_mi_clflush},
+
+        {"MI_REPORT_PERF_COUNT", OP_MI_REPORT_PERF_COUNT, F_LEN_VAR, R_ALL,
+                D_ALL, ADDR_FIX_1(1), 6, cmd_handler_mi_report_perf_count},
+
+        {"MI_LOAD_REGISTER_MEM", OP_MI_LOAD_REGISTER_MEM, F_LEN_VAR, R_ALL,
+                D_ALL, ADDR_FIX_1(2), 8, cmd_handler_lrm},
+
+        {"MI_LOAD_REGISTER_REG", OP_MI_LOAD_REGISTER_REG, F_LEN_VAR, R_ALL,
+                D_ALL, 0, 8, cmd_handler_lrr},
+
+        {"MI_RS_STORE_DATA_IMM", OP_MI_RS_STORE_DATA_IMM, F_LEN_VAR, R_RCS,
+                D_ALL, 0, 8, NULL},
+
+        {"MI_LOAD_URB_MEM", OP_MI_LOAD_URB_MEM, F_LEN_VAR, R_RCS, D_ALL,
+                ADDR_FIX_1(2), 8, NULL},
+
+        {"MI_STORE_URM_MEM", OP_MI_STORE_URM_MEM, F_LEN_VAR, R_RCS, D_ALL,
+                ADDR_FIX_1(2), 8, NULL},
+
+        {"MI_OP_2E", OP_MI_2E, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_2(1, 2),
+                8, cmd_handler_mi_op_2e},
+
+        {"MI_OP_2F", OP_MI_2F, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1),
+                8, cmd_handler_mi_op_2f},
+
+        {"MI_BATCH_BUFFER_START", OP_MI_BATCH_BUFFER_START,
+                F_IP_ADVANCE_CUSTOM, R_ALL, D_ALL, 0, 8,
+                cmd_handler_mi_batch_buffer_start},
+
+        {"MI_CONDITIONAL_BATCH_BUFFER_END", OP_MI_CONDITIONAL_BATCH_BUFFER_END,
+                F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(2), 8,
+                cmd_handler_mi_conditional_batch_buffer_end},
+
+        {"MI_LOAD_SCAN_LINES_INCL", OP_MI_LOAD_SCAN_LINES_INCL, F_LEN_CONST,
+                R_RCS | R_BCS, D_ALL, 0, 2, NULL},
+
+        {"XY_SETUP_BLT", OP_XY_SETUP_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                ADDR_FIX_2(4, 7), 8, NULL},
+
+        {"XY_SETUP_CLIP_BLT", OP_XY_SETUP_CLIP_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                0, 8, NULL},
+
+        {"XY_SETUP_MONO_PATTERN_SL_BLT", OP_XY_SETUP_MONO_PATTERN_SL_BLT,
+                F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+        {"XY_PIXEL_BLT", OP_XY_PIXEL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
+
+        {"XY_SCANLINES_BLT", OP_XY_SCANLINES_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                0, 8, NULL},
+
+        {"XY_TEXT_BLT", OP_XY_TEXT_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                ADDR_FIX_1(3), 8, NULL},
+
+        {"XY_TEXT_IMMEDIATE_BLT", OP_XY_TEXT_IMMEDIATE_BLT, F_LEN_VAR, R_BCS,
+                D_ALL, 0, 8, NULL},
+
+        {"XY_COLOR_BLT", OP_XY_COLOR_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                ADDR_FIX_1(4), 8, NULL},
+
+        {"XY_PAT_BLT", OP_XY_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                ADDR_FIX_2(4, 5), 8, NULL},
+
+        {"XY_MONO_PAT_BLT", OP_XY_MONO_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                ADDR_FIX_1(4), 8, NULL},
+
+        {"XY_SRC_COPY_BLT", OP_XY_SRC_COPY_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                ADDR_FIX_2(4, 7), 8, NULL},
+
+        {"XY_MONO_SRC_COPY_BLT", OP_XY_MONO_SRC_COPY_BLT, F_LEN_VAR, R_BCS,
+                D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
+
+        {"XY_FULL_BLT", OP_XY_FULL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
+
+        {"XY_FULL_MONO_SRC_BLT", OP_XY_FULL_MONO_SRC_BLT, F_LEN_VAR, R_BCS,
+                D_ALL, ADDR_FIX_3(4, 5, 8), 8, NULL},
+
+        {"XY_FULL_MONO_PATTERN_BLT", OP_XY_FULL_MONO_PATTERN_BLT, F_LEN_VAR,
+                R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
+
+        {"XY_FULL_MONO_PATTERN_MONO_SRC_BLT",
+                OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT,
+                F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
+
+        {"XY_MONO_PAT_FIXED_BLT", OP_XY_MONO_PAT_FIXED_BLT, F_LEN_VAR, R_BCS,
+                D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+        {"XY_MONO_SRC_COPY_IMMEDIATE_BLT", OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT,
+                F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+        {"XY_PAT_BLT_IMMEDIATE", OP_XY_PAT_BLT_IMMEDIATE, F_LEN_VAR, R_BCS,
+                D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+        {"XY_SRC_COPY_CHROMA_BLT", OP_XY_SRC_COPY_CHROMA_BLT, F_LEN_VAR, R_BCS,
+                D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
+
+        {"XY_FULL_IMMEDIATE_PATTERN_BLT", OP_XY_FULL_IMMEDIATE_PATTERN_BLT,
+                F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
+
+        {"XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT",
+                OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT,
+                F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
+
+        {"XY_PAT_CHROMA_BLT", OP_XY_PAT_CHROMA_BLT, F_LEN_VAR, R_BCS, D_ALL,
+                ADDR_FIX_2(4, 5), 8, NULL},
+
+        {"XY_PAT_CHROMA_BLT_IMMEDIATE", OP_XY_PAT_CHROMA_BLT_IMMEDIATE,
+                F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+        {"3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP",
+                OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_VIEWPORT_STATE_POINTERS_CC",
+                OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_BLEND_STATE_POINTERS",
+                OP_3DSTATE_BLEND_STATE_POINTERS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DEPTH_STENCIL_STATE_POINTERS",
+                OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_BINDING_TABLE_POINTERS_VS",
+                OP_3DSTATE_BINDING_TABLE_POINTERS_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_BINDING_TABLE_POINTERS_HS",
+                OP_3DSTATE_BINDING_TABLE_POINTERS_HS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_BINDING_TABLE_POINTERS_DS",
+                OP_3DSTATE_BINDING_TABLE_POINTERS_DS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_BINDING_TABLE_POINTERS_GS",
+                OP_3DSTATE_BINDING_TABLE_POINTERS_GS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_BINDING_TABLE_POINTERS_PS",
+                OP_3DSTATE_BINDING_TABLE_POINTERS_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLER_STATE_POINTERS_VS",
+                OP_3DSTATE_SAMPLER_STATE_POINTERS_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLER_STATE_POINTERS_HS",
+                OP_3DSTATE_SAMPLER_STATE_POINTERS_HS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLER_STATE_POINTERS_DS",
+                OP_3DSTATE_SAMPLER_STATE_POINTERS_DS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLER_STATE_POINTERS_GS",
+                OP_3DSTATE_SAMPLER_STATE_POINTERS_GS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLER_STATE_POINTERS_PS",
+                OP_3DSTATE_SAMPLER_STATE_POINTERS_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_URB_VS", OP_3DSTATE_URB_VS, F_LEN_VAR, R_RCS, D_ALL,
+                0, 8, NULL},
+
+        {"3DSTATE_URB_HS", OP_3DSTATE_URB_HS, F_LEN_VAR, R_RCS, D_ALL,
+                0, 8, NULL},
+
+        {"3DSTATE_URB_DS", OP_3DSTATE_URB_DS, F_LEN_VAR, R_RCS, D_ALL,
+                0, 8, NULL},
+
+        {"3DSTATE_URB_GS", OP_3DSTATE_URB_GS, F_LEN_VAR, R_RCS, D_ALL,
+                0, 8, NULL},
+
+        {"3DSTATE_GATHER_CONSTANT_VS", OP_3DSTATE_GATHER_CONSTANT_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_GATHER_CONSTANT_GS", OP_3DSTATE_GATHER_CONSTANT_GS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_GATHER_CONSTANT_HS", OP_3DSTATE_GATHER_CONSTANT_HS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_GATHER_CONSTANT_DS", OP_3DSTATE_GATHER_CONSTANT_DS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_GATHER_CONSTANT_PS", OP_3DSTATE_GATHER_CONSTANT_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_CONSTANTF_VS", OP_3DSTATE_DX9_CONSTANTF_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
+
+        {"3DSTATE_DX9_CONSTANTF_PS", OP_3DSTATE_DX9_CONSTANTF_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
+
+        {"3DSTATE_DX9_CONSTANTI_VS", OP_3DSTATE_DX9_CONSTANTI_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_CONSTANTI_PS", OP_3DSTATE_DX9_CONSTANTI_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_CONSTANTB_VS", OP_3DSTATE_DX9_CONSTANTB_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_CONSTANTB_PS", OP_3DSTATE_DX9_CONSTANTB_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_LOCAL_VALID_VS", OP_3DSTATE_DX9_LOCAL_VALID_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_LOCAL_VALID_PS", OP_3DSTATE_DX9_LOCAL_VALID_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_GENERATE_ACTIVE_VS", OP_3DSTATE_DX9_GENERATE_ACTIVE_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DX9_GENERATE_ACTIVE_PS", OP_3DSTATE_DX9_GENERATE_ACTIVE_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_BINDING_TABLE_EDIT_VS", OP_3DSTATE_BINDING_TABLE_EDIT_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+        {"3DSTATE_BINDING_TABLE_EDIT_GS", OP_3DSTATE_BINDING_TABLE_EDIT_GS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+        {"3DSTATE_BINDING_TABLE_EDIT_HS", OP_3DSTATE_BINDING_TABLE_EDIT_HS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+        {"3DSTATE_BINDING_TABLE_EDIT_DS", OP_3DSTATE_BINDING_TABLE_EDIT_DS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+        {"3DSTATE_BINDING_TABLE_EDIT_PS", OP_3DSTATE_BINDING_TABLE_EDIT_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+        {"3DSTATE_VF_INSTANCING", OP_3DSTATE_VF_INSTANCING, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_VF_SGVS", OP_3DSTATE_VF_SGVS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+                NULL},
+
+        {"3DSTATE_VF_TOPOLOGY", OP_3DSTATE_VF_TOPOLOGY, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_WM_CHROMAKEY", OP_3DSTATE_WM_CHROMAKEY, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_PS_BLEND", OP_3DSTATE_PS_BLEND, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
+                8, NULL},
+
+        {"3DSTATE_WM_DEPTH_STENCIL", OP_3DSTATE_WM_DEPTH_STENCIL, F_LEN_VAR,
+                R_RCS, D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_PS_EXTRA", OP_3DSTATE_PS_EXTRA, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
+                8, NULL},
+
+        {"3DSTATE_RASTER", OP_3DSTATE_RASTER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+                NULL},
+
+        {"3DSTATE_SBE_SWIZ", OP_3DSTATE_SBE_SWIZ, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+                NULL},
+
+        {"3DSTATE_WM_HZ_OP", OP_3DSTATE_WM_HZ_OP, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+                NULL},
+
+        {"3DSTATE_VERTEX_BUFFERS", OP_3DSTATE_VERTEX_BUFFERS, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_VERTEX_ELEMENTS", OP_3DSTATE_VERTEX_ELEMENTS, F_LEN_VAR,
+                R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_INDEX_BUFFER", OP_3DSTATE_INDEX_BUFFER, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, ADDR_FIX_1(2), 8, NULL},
+
+        {"3DSTATE_VF_STATISTICS", OP_3DSTATE_VF_STATISTICS, F_LEN_CONST,
+                R_RCS, D_ALL, 0, 1, NULL},
+
+        {"3DSTATE_VF", OP_3DSTATE_VF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_CC_STATE_POINTERS", OP_3DSTATE_CC_STATE_POINTERS, F_LEN_VAR,
+                R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SCISSOR_STATE_POINTERS", OP_3DSTATE_SCISSOR_STATE_POINTERS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_GS", OP_3DSTATE_GS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_CLIP", OP_3DSTATE_CLIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_WM", OP_3DSTATE_WM, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_CONSTANT_GS", OP_3DSTATE_CONSTANT_GS, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_CONSTANT_PS", OP_3DSTATE_CONSTANT_PS, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLE_MASK", OP_3DSTATE_SAMPLE_MASK, F_LEN_VAR, R_RCS,
+                D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_CONSTANT_HS", OP_3DSTATE_CONSTANT_HS, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_CONSTANT_DS", OP_3DSTATE_CONSTANT_DS, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_HS", OP_3DSTATE_HS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_TE", OP_3DSTATE_TE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DS", OP_3DSTATE_DS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_STREAMOUT", OP_3DSTATE_STREAMOUT, F_LEN_VAR, R_RCS,
+                D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SBE", OP_3DSTATE_SBE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_PS", OP_3DSTATE_PS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_DRAWING_RECTANGLE", OP_3DSTATE_DRAWING_RECTANGLE, F_LEN_VAR,
+                R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLER_PALETTE_LOAD0", OP_3DSTATE_SAMPLER_PALETTE_LOAD0,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_CHROMA_KEY", OP_3DSTATE_CHROMA_KEY, F_LEN_VAR, R_RCS, D_ALL,
+                0, 8, NULL},
+
+        {"3DSTATE_DEPTH_BUFFER", OP_3DSTATE_DEPTH_BUFFER, F_LEN_VAR, R_RCS,
+                D_ALL, ADDR_FIX_1(2), 8, NULL},
+
+        {"3DSTATE_POLY_STIPPLE_OFFSET", OP_3DSTATE_POLY_STIPPLE_OFFSET,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_POLY_STIPPLE_PATTERN", OP_3DSTATE_POLY_STIPPLE_PATTERN,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_LINE_STIPPLE", OP_3DSTATE_LINE_STIPPLE, F_LEN_VAR, R_RCS,
+                D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_AA_LINE_PARAMS", OP_3DSTATE_AA_LINE_PARAMS, F_LEN_VAR, R_RCS,
+                D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_GS_SVB_INDEX", OP_3DSTATE_GS_SVB_INDEX, F_LEN_VAR, R_RCS,
+                D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SAMPLER_PALETTE_LOAD1", OP_3DSTATE_SAMPLER_PALETTE_LOAD1,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_MULTISAMPLE", OP_3DSTATE_MULTISAMPLE_BDW, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"3DSTATE_STENCIL_BUFFER", OP_3DSTATE_STENCIL_BUFFER, F_LEN_VAR, R_RCS,
+                D_ALL, ADDR_FIX_1(2), 8, NULL},
+
+        {"3DSTATE_HIER_DEPTH_BUFFER", OP_3DSTATE_HIER_DEPTH_BUFFER, F_LEN_VAR,
+                R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL},
+
+        {"3DSTATE_CLEAR_PARAMS", OP_3DSTATE_CLEAR_PARAMS, F_LEN_VAR,
+                R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_PUSH_CONSTANT_ALLOC_VS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_PUSH_CONSTANT_ALLOC_HS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_PUSH_CONSTANT_ALLOC_DS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_PUSH_CONSTANT_ALLOC_GS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_PUSH_CONSTANT_ALLOC_PS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_MONOFILTER_SIZE", OP_3DSTATE_MONOFILTER_SIZE, F_LEN_VAR,
+                R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SO_DECL_LIST", OP_3DSTATE_SO_DECL_LIST, F_LEN_VAR, R_RCS,
+                D_ALL, 0, 9, NULL},
+
+        {"3DSTATE_SO_BUFFER", OP_3DSTATE_SO_BUFFER, F_LEN_VAR, R_RCS, D_BDW_PLUS,
+                ADDR_FIX_2(2, 4), 8, NULL},
+
+        {"3DSTATE_BINDING_TABLE_POOL_ALLOC",
+                OP_3DSTATE_BINDING_TABLE_POOL_ALLOC,
+                F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
+
+        {"3DSTATE_GATHER_POOL_ALLOC", OP_3DSTATE_GATHER_POOL_ALLOC,
+                F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
+
+        {"3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC",
+                OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC,
+                F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
+
+        {"3DSTATE_SAMPLE_PATTERN", OP_3DSTATE_SAMPLE_PATTERN, F_LEN_VAR, R_RCS,
+                D_BDW_PLUS, 0, 8, NULL},
+
+        {"PIPE_CONTROL", OP_PIPE_CONTROL, F_LEN_VAR, R_RCS, D_ALL,
+                ADDR_FIX_1(2), 8, cmd_handler_pipe_control},
+
+        {"3DPRIMITIVE", OP_3DPRIMITIVE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"PIPELINE_SELECT", OP_PIPELINE_SELECT, F_LEN_CONST, R_RCS, D_ALL, 0,
+                1, NULL},
+
+        {"STATE_PREFETCH", OP_STATE_PREFETCH, F_LEN_VAR, R_RCS, D_ALL,
+                ADDR_FIX_1(1), 8, NULL},
+
+        {"STATE_SIP", OP_STATE_SIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"STATE_BASE_ADDRESS", OP_STATE_BASE_ADDRESS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
+                ADDR_FIX_5(1, 3, 4, 5, 6), 8, NULL},
+
+        {"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL,
+                ADDR_FIX_1(1), 8, NULL},
+
+        {"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+        {"3DSTATE_CONSTANT_VS", OP_3DSTATE_CONSTANT_VS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
+                0, 8, NULL},
+
+        {"3DSTATE_COMPONENT_PACKING", OP_3DSTATE_COMPONENT_PACKING, F_LEN_VAR, R_RCS,
+                D_SKL_PLUS, 0, 8, NULL},
+
+        {"MEDIA_INTERFACE_DESCRIPTOR_LOAD", OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
+                F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
+
+        {"MEDIA_GATEWAY_STATE", OP_MEDIA_GATEWAY_STATE, F_LEN_VAR, R_RCS, D_ALL,
+                0, 16, NULL},
+
+        {"MEDIA_STATE_FLUSH", OP_MEDIA_STATE_FLUSH, F_LEN_VAR, R_RCS, D_ALL,
+                0, 16, NULL},
+
+        {"MEDIA_OBJECT", OP_MEDIA_OBJECT, F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
+
+        {"MEDIA_CURBE_LOAD", OP_MEDIA_CURBE_LOAD, F_LEN_VAR, R_RCS, D_ALL,
+                0, 16, NULL},
+
+        {"MEDIA_OBJECT_PRT", OP_MEDIA_OBJECT_PRT, F_LEN_VAR, R_RCS, D_ALL,
+                0, 16, NULL},
+
+        {"MEDIA_OBJECT_WALKER", OP_MEDIA_OBJECT_WALKER, F_LEN_VAR, R_RCS, D_ALL,
+                0, 16, NULL},
+
+        {"GPGPU_WALKER", OP_GPGPU_WALKER, F_LEN_VAR, R_RCS, D_ALL,
+                0, 8, NULL},
+
+        {"MEDIA_VFE_STATE", OP_MEDIA_VFE_STATE, F_LEN_VAR, R_RCS, D_ALL, 0, 16,
+                NULL},
+
+        {"3DSTATE_VF_STATISTICS_GM45", OP_3DSTATE_VF_STATISTICS_GM45,
+                F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
+
+        {"MFX_PIPE_MODE_SELECT", OP_MFX_PIPE_MODE_SELECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_SURFACE_STATE", OP_MFX_SURFACE_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_PIPE_BUF_ADDR_STATE", OP_MFX_PIPE_BUF_ADDR_STATE, F_LEN_VAR,
+                R_VCS, D_BDW_PLUS, 0, 12, NULL},
+
+        {"MFX_IND_OBJ_BASE_ADDR_STATE", OP_MFX_IND_OBJ_BASE_ADDR_STATE,
+                F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
+
+        {"MFX_BSP_BUF_BASE_ADDR_STATE", OP_MFX_BSP_BUF_BASE_ADDR_STATE,
+                F_LEN_VAR, R_VCS, D_BDW_PLUS, ADDR_FIX_3(1, 3, 5), 12, NULL},
+
+        {"OP_2_0_0_5", OP_2_0_0_5, F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
+
+        {"MFX_STATE_POINTER", OP_MFX_STATE_POINTER, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_QM_STATE", OP_MFX_QM_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_FQM_STATE", OP_MFX_FQM_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_PAK_INSERT_OBJECT", OP_MFX_PAK_INSERT_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_STITCH_OBJECT", OP_MFX_STITCH_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_IT_OBJECT", OP_MFD_IT_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_WAIT", OP_MFX_WAIT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 6, NULL},
+
+        {"MFX_AVC_IMG_STATE", OP_MFX_AVC_IMG_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_AVC_QM_STATE", OP_MFX_AVC_QM_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_AVC_DIRECTMODE_STATE", OP_MFX_AVC_DIRECTMODE_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_AVC_SLICE_STATE", OP_MFX_AVC_SLICE_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_AVC_REF_IDX_STATE", OP_MFX_AVC_REF_IDX_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_AVC_WEIGHTOFFSET_STATE", OP_MFX_AVC_WEIGHTOFFSET_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_AVC_PICID_STATE", OP_MFD_AVC_PICID_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+        {"MFD_AVC_DPB_STATE", OP_MFD_AVC_DPB_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_AVC_BSD_OBJECT", OP_MFD_AVC_BSD_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_AVC_SLICEADDR", OP_MFD_AVC_SLICEADDR, F_LEN_VAR,
+                R_VCS, D_ALL, ADDR_FIX_1(2), 12, NULL},
+
+        {"MFC_AVC_PAK_OBJECT", OP_MFC_AVC_PAK_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_VC1_PRED_PIPE_STATE", OP_MFX_VC1_PRED_PIPE_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_VC1_DIRECTMODE_STATE", OP_MFX_VC1_DIRECTMODE_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_VC1_SHORT_PIC_STATE", OP_MFD_VC1_SHORT_PIC_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_VC1_LONG_PIC_STATE", OP_MFD_VC1_LONG_PIC_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_VC1_BSD_OBJECT", OP_MFD_VC1_BSD_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFC_MPEG2_SLICEGROUP_STATE", OP_MFC_MPEG2_SLICEGROUP_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFC_MPEG2_PAK_OBJECT", OP_MFC_MPEG2_PAK_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_MPEG2_PIC_STATE", OP_MFX_MPEG2_PIC_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_MPEG2_QM_STATE", OP_MFX_MPEG2_QM_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_MPEG2_BSD_OBJECT", OP_MFD_MPEG2_BSD_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_2_6_0_0", OP_MFX_2_6_0_0, F_LEN_VAR, R_VCS, D_ALL,
+                0, 16, NULL},
+
+        {"MFX_2_6_0_9", OP_MFX_2_6_0_9, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
+
+        {"MFX_2_6_0_8", OP_MFX_2_6_0_8, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
+
+        {"MFX_JPEG_PIC_STATE", OP_MFX_JPEG_PIC_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFX_JPEG_HUFF_TABLE_STATE", OP_MFX_JPEG_HUFF_TABLE_STATE, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"MFD_JPEG_BSD_OBJECT", OP_MFD_JPEG_BSD_OBJECT, F_LEN_VAR,
+                R_VCS, D_ALL, 0, 12, NULL},
+
+        {"VEBOX_STATE", OP_VEB_STATE, F_LEN_VAR, R_VECS, D_ALL, 0, 12, NULL},
+
+        {"VEBOX_SURFACE_STATE", OP_VEB_SURFACE_STATE, F_LEN_VAR, R_VECS, D_ALL,
+                0, 12, NULL},
+
+        {"VEB_DI_IECP", OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS,
+                0, 20, NULL},
+};
+
+static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e)
+{
+        hash_add(gvt->cmd_table, &e->hlist, e->info->opcode);
+}
+
+#define GVT_MAX_CMD_LENGTH     20  /* In Dword */
+
+static void trace_cs_command(struct parser_exec_state *s,
+                cycles_t cost_pre_cmd_handler, cycles_t cost_cmd_handler)
+{
+        /* This buffer is used by ftrace to store all commands copied from
+         * guest gma space. Sometimes commands can cross pages, this should
+         * not be handled in ftrace logic. So this is just used as a
+         * 'bounce buffer'
+         */
+        u32 cmd_trace_buf[GVT_MAX_CMD_LENGTH];
+        int i;
+        u32 cmd_len = cmd_length(s);
+        /* The chosen value of GVT_MAX_CMD_LENGTH are just based on
+         * following two considerations:
+         * 1) From observation, most common ring commands is not that long.
+         *    But there are execeptions. So it indeed makes sence to observe
+         *    longer commands.
+         * 2) From the performance and debugging point of view, dumping all
+         *    contents of very commands is not necessary.
+         * We mgith shrink GVT_MAX_CMD_LENGTH or remove this trace event in
+         * future for performance considerations.
+         */
+        if (unlikely(cmd_len > GVT_MAX_CMD_LENGTH)) {
+                gvt_dbg_cmd("cmd length exceed tracing limitation!\n");
+                cmd_len = GVT_MAX_CMD_LENGTH;
+        }
+
+        for (i = 0; i < cmd_len; i++)
+                cmd_trace_buf[i] = cmd_val(s, i);
+
+        trace_gvt_command(s->vgpu->id, s->ring_id, s->ip_gma, cmd_trace_buf,
+                        cmd_len, s->buf_type == RING_BUFFER_INSTRUCTION,
+                        cost_pre_cmd_handler, cost_cmd_handler);
+}
+
+/* call the cmd handler, and advance ip */
+static int cmd_parser_exec(struct parser_exec_state *s)
+{
+        struct cmd_info *info;
+        u32 cmd;
+        int ret = 0;
+        cycles_t t0, t1, t2;
+        struct parser_exec_state s_before_advance_custom;
+
+        t0 = get_cycles();
+
+        cmd = cmd_val(s, 0);
+
+        info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
+        if (info == NULL) {
+                gvt_err("unknown cmd 0x%x, opcode=0x%x\n",
+                                cmd, get_opcode(cmd, s->ring_id));
+                return -EINVAL;
+        }
+
+        gvt_dbg_cmd("%s\n", info->name);
+
+        s->info = info;
+
+        t1 = get_cycles();
+
+        memcpy(&s_before_advance_custom, s, sizeof(struct parser_exec_state));
+
+        if (info->handler) {
+                ret = info->handler(s);
+                if (ret < 0) {
+                        gvt_err("%s handler error\n", info->name);
+                        return ret;
+                }
+        }
+        t2 = get_cycles();
+
+        trace_cs_command(&s_before_advance_custom, t1 - t0, t2 - t1);
+
+        if (!(info->flag & F_IP_ADVANCE_CUSTOM)) {
+                ret = cmd_advance_default(s);
+                if (ret) {
+                        gvt_err("%s IP advance error\n", info->name);
+                        return ret;
+                }
+        }
+        return 0;
+}
+
+static inline bool gma_out_of_range(unsigned long gma,
+                unsigned long gma_head, unsigned int gma_tail)
+{
+        if (gma_tail >= gma_head)
+                return (gma < gma_head) || (gma > gma_tail);
+        else
+                return (gma > gma_tail) && (gma < gma_head);
+}
+
+static int command_scan(struct parser_exec_state *s,
+                unsigned long rb_head, unsigned long rb_tail,
+                unsigned long rb_start, unsigned long rb_len)
+{
+
+        unsigned long gma_head, gma_tail, gma_bottom;
+        int ret = 0;
+
+        gma_head = rb_start + rb_head;
+        gma_tail = rb_start + rb_tail;
+        gma_bottom = rb_start +  rb_len;
+
+        gvt_dbg_cmd("scan_start: start=%lx end=%lx\n", gma_head, gma_tail);
+
+        while (s->ip_gma != gma_tail) {
+                if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+                        if (!(s->ip_gma >= rb_start) ||
+                                !(s->ip_gma < gma_bottom)) {
+                                gvt_err("ip_gma %lx out of ring scope."
+                                        "(base:0x%lx, bottom: 0x%lx)\n",
+                                        s->ip_gma, rb_start,
+                                        gma_bottom);
+                                parser_exec_state_dump(s);
+                                return -EINVAL;
+                        }
+                        if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) {
+                                gvt_err("ip_gma %lx out of range."
+                                        "base 0x%lx head 0x%lx tail 0x%lx\n",
+                                        s->ip_gma, rb_start,
+                                        rb_head, rb_tail);
+                                parser_exec_state_dump(s);
+                                break;
+                        }
+                }
+                ret = cmd_parser_exec(s);
+                if (ret) {
+                        gvt_err("cmd parser error\n");
+                        parser_exec_state_dump(s);
+                        break;
+                }
+        }
+
+        gvt_dbg_cmd("scan_end\n");
+
+        return ret;
+}
+
+static int scan_workload(struct intel_vgpu_workload *workload)
+{
+        unsigned long gma_head, gma_tail, gma_bottom;
+        struct parser_exec_state s;
+        int ret = 0;
+
+        /* ring base is page aligned */
+        if (WARN_ON(!IS_ALIGNED(workload->rb_start, GTT_PAGE_SIZE)))
+                return -EINVAL;
+
+        gma_head = workload->rb_start + workload->rb_head;
+        gma_tail = workload->rb_start + workload->rb_tail;
+        gma_bottom = workload->rb_start +  _RING_CTL_BUF_SIZE(workload->rb_ctl);
+
+        s.buf_type = RING_BUFFER_INSTRUCTION;
+        s.buf_addr_type = GTT_BUFFER;
+        s.vgpu = workload->vgpu;
+        s.ring_id = workload->ring_id;
+        s.ring_start = workload->rb_start;
+        s.ring_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
+        s.ring_head = gma_head;
+        s.ring_tail = gma_tail;
+        s.rb_va = workload->shadow_ring_buffer_va;
+        s.workload = workload;
+
+        if (bypass_scan_mask & (1 << workload->ring_id))
+                return 0;
+
+        ret = ip_gma_set(&s, gma_head);
+        if (ret)
+                goto out;
+
+        ret = command_scan(&s, workload->rb_head, workload->rb_tail,
+                workload->rb_start, _RING_CTL_BUF_SIZE(workload->rb_ctl));
+
+out:
+        return ret;
+}
+
+static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+
+        unsigned long gma_head, gma_tail, gma_bottom, ring_size, ring_tail;
+        struct parser_exec_state s;
+        int ret = 0;
+
+        /* ring base is page aligned */
+        if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, GTT_PAGE_SIZE)))
+                return -EINVAL;
+
+        ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(uint32_t);
+        ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES,
+                        PAGE_SIZE);
+        gma_head = wa_ctx->indirect_ctx.guest_gma;
+        gma_tail = wa_ctx->indirect_ctx.guest_gma + ring_tail;
+        gma_bottom = wa_ctx->indirect_ctx.guest_gma + ring_size;
+
+        s.buf_type = RING_BUFFER_INSTRUCTION;
+        s.buf_addr_type = GTT_BUFFER;
+        s.vgpu = wa_ctx->workload->vgpu;
+        s.ring_id = wa_ctx->workload->ring_id;
+        s.ring_start = wa_ctx->indirect_ctx.guest_gma;
+        s.ring_size = ring_size;
+        s.ring_head = gma_head;
+        s.ring_tail = gma_tail;
+        s.rb_va = wa_ctx->indirect_ctx.shadow_va;
+        s.workload = wa_ctx->workload;
+
+        ret = ip_gma_set(&s, gma_head);
+        if (ret)
+                goto out;
+
+        ret = command_scan(&s, 0, ring_tail,
+                wa_ctx->indirect_ctx.guest_gma, ring_size);
+out:
+        return ret;
+}
+
+static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        int ring_id = workload->ring_id;
+        struct i915_gem_context *shadow_ctx = vgpu->shadow_ctx;
+        struct intel_ring *ring = shadow_ctx->engine[ring_id].ring;
+        unsigned long gma_head, gma_tail, gma_top, guest_rb_size;
+        unsigned int copy_len = 0;
+        int ret;
+
+        guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
+
+        /* calculate workload ring buffer size */
+        workload->rb_len = (workload->rb_tail + guest_rb_size -
+                        workload->rb_head) % guest_rb_size;
+
+        gma_head = workload->rb_start + workload->rb_head;
+        gma_tail = workload->rb_start + workload->rb_tail;
+        gma_top = workload->rb_start + guest_rb_size;
+
+        /* allocate shadow ring buffer */
+        ret = intel_ring_begin(workload->req, workload->rb_len / 4);
+        if (ret)
+                return ret;
+
+        /* get shadow ring buffer va */
+        workload->shadow_ring_buffer_va = ring->vaddr + ring->tail;
+
+        /* head > tail --> copy head <-> top */
+        if (gma_head > gma_tail) {
+                ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm,
+                                gma_head, gma_top,
+                                workload->shadow_ring_buffer_va);
+                if (ret) {
+                        gvt_err("fail to copy guest ring buffer\n");
+                        return ret;
+                }
+                copy_len = gma_top - gma_head;
+                gma_head = workload->rb_start;
+        }
+
+        /* copy head or start <-> tail */
+        ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm,
+                        gma_head, gma_tail,
+                        workload->shadow_ring_buffer_va + copy_len);
+        if (ret) {
+                gvt_err("fail to copy guest ring buffer\n");
+                return ret;
+        }
+        ring->tail += workload->rb_len;
+        intel_ring_advance(ring);
+        return 0;
+}
+
+int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
+{
+        int ret;
+
+        ret = shadow_workload_ring_buffer(workload);
+        if (ret) {
+                gvt_err("fail to shadow workload ring_buffer\n");
+                return ret;
+        }
+
+        ret = scan_workload(workload);
+        if (ret) {
+                gvt_err("scan workload error\n");
+                return ret;
+        }
+        return 0;
+}
+
+static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        struct drm_device *dev = &wa_ctx->workload->vgpu->gvt->dev_priv->drm;
+        int ctx_size = wa_ctx->indirect_ctx.size;
+        unsigned long guest_gma = wa_ctx->indirect_ctx.guest_gma;
+        struct drm_i915_gem_object *obj;
+        int ret = 0;
+        void *map;
+
+        obj = i915_gem_object_create(dev,
+                                     roundup(ctx_size + CACHELINE_BYTES,
+                                             PAGE_SIZE));
+        if (IS_ERR(obj))
+                return PTR_ERR(obj);
+
+        /* get the va of the shadow batch buffer */
+        map = i915_gem_object_pin_map(obj, I915_MAP_WB);
+        if (IS_ERR(map)) {
+                gvt_err("failed to vmap shadow indirect ctx\n");
+                ret = PTR_ERR(map);
+                goto put_obj;
+        }
+
+        ret = i915_gem_object_set_to_cpu_domain(obj, false);
+        if (ret) {
+                gvt_err("failed to set shadow indirect ctx to CPU\n");
+                goto unmap_src;
+        }
+
+        ret = copy_gma_to_hva(wa_ctx->workload->vgpu,
+                                wa_ctx->workload->vgpu->gtt.ggtt_mm,
+                                guest_gma, guest_gma + ctx_size,
+                                map);
+        if (ret) {
+                gvt_err("fail to copy guest indirect ctx\n");
+                goto unmap_src;
+        }
+
+        wa_ctx->indirect_ctx.obj = obj;
+        wa_ctx->indirect_ctx.shadow_va = map;
+        return 0;
+
+unmap_src:
+        i915_gem_object_unpin_map(obj);
+put_obj:
+        i915_gem_object_put(wa_ctx->indirect_ctx.obj);
+        return ret;
+}
+
+static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        uint32_t per_ctx_start[CACHELINE_DWORDS] = {0};
+        unsigned char *bb_start_sva;
+
+        per_ctx_start[0] = 0x18800001;
+        per_ctx_start[1] = wa_ctx->per_ctx.guest_gma;
+
+        bb_start_sva = (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
+                                wa_ctx->indirect_ctx.size;
+
+        memcpy(bb_start_sva, per_ctx_start, CACHELINE_BYTES);
+
+        return 0;
+}
+
+int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        int ret;
+
+        if (wa_ctx->indirect_ctx.size == 0)
+                return 0;
+
+        ret = shadow_indirect_ctx(wa_ctx);
+        if (ret) {
+                gvt_err("fail to shadow indirect ctx\n");
+                return ret;
+        }
+
+        combine_wa_ctx(wa_ctx);
+
+        ret = scan_wa_ctx(wa_ctx);
+        if (ret) {
+                gvt_err("scan wa ctx error\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+static struct cmd_info *find_cmd_entry_any_ring(struct intel_gvt *gvt,
+                unsigned int opcode, int rings)
+{
+        struct cmd_info *info = NULL;
+        unsigned int ring;
+
+        for_each_set_bit(ring, (unsigned long *)&rings, I915_NUM_ENGINES) {
+                info = find_cmd_entry(gvt, opcode, ring);
+                if (info)
+                        break;
+        }
+        return info;
+}
+
+static int init_cmd_table(struct intel_gvt *gvt)
+{
+        int i;
+        struct cmd_entry *e;
+        struct cmd_info *info;
+        unsigned int gen_type;
+
+        gen_type = intel_gvt_get_device_type(gvt);
+
+        for (i = 0; i < ARRAY_SIZE(cmd_info); i++) {
+                if (!(cmd_info[i].devices & gen_type))
+                        continue;
+
+                e = kzalloc(sizeof(*e), GFP_KERNEL);
+                if (!e)
+                        return -ENOMEM;
+
+                e->info = &cmd_info[i];
+                info = find_cmd_entry_any_ring(gvt,
+                                e->info->opcode, e->info->rings);
+                if (info) {
+                        gvt_err("%s %s duplicated\n", e->info->name,
+                                        info->name);
+                        return -EEXIST;
+                }
+
+                INIT_HLIST_NODE(&e->hlist);
+                add_cmd_entry(gvt, e);
+                gvt_dbg_cmd("add %-30s op %04x flag %x devs %02x rings %02x\n",
+                                e->info->name, e->info->opcode, e->info->flag,
+                                e->info->devices, e->info->rings);
+        }
+        return 0;
+}
+
+static void clean_cmd_table(struct intel_gvt *gvt)
+{
+        struct hlist_node *tmp;
+        struct cmd_entry *e;
+        int i;
+
+        hash_for_each_safe(gvt->cmd_table, i, tmp, e, hlist)
+                kfree(e);
+
+        hash_init(gvt->cmd_table);
+}
+
+void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt)
+{
+        clean_cmd_table(gvt);
+}
+
+int intel_gvt_init_cmd_parser(struct intel_gvt *gvt)
+{
+        int ret;
+
+        ret = init_cmd_table(gvt);
+        if (ret) {
+                intel_gvt_clean_cmd_parser(gvt);
+                return ret;
+        }
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.h b/drivers/gpu/drm/i915/gvt/cmd_parser.h
new file mode 100644
index 000000000000..bed33514103c
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Yulei Zhang <yulei.zhang@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+#ifndef _GVT_CMD_PARSER_H_
+#define _GVT_CMD_PARSER_H_
+
+#define GVT_CMD_HASH_BITS 7
+
+void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt);
+
+int intel_gvt_init_cmd_parser(struct intel_gvt *gvt);
+
+int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
+
+int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/debug.h b/drivers/gpu/drm/i915/gvt/debug.h
index 7ef412be665f..68cba7bd980a 100644
--- a/drivers/gpu/drm/i915/gvt/debug.h
+++ b/drivers/gpu/drm/i915/gvt/debug.h
@@ -24,11 +24,34 @@
 #ifndef __GVT_DEBUG_H__
 #define __GVT_DEBUG_H__
 
+#define gvt_err(fmt, args...) \
+        DRM_ERROR("gvt: "fmt, ##args)
+
 #define gvt_dbg_core(fmt, args...) \
         DRM_DEBUG_DRIVER("gvt: core: "fmt, ##args)
 
-/*
- * Other GVT debug stuff will be introduced in the GVT device model patches.
- */
+#define gvt_dbg_irq(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: irq: "fmt, ##args)
+
+#define gvt_dbg_mm(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: mm: "fmt, ##args)
+
+#define gvt_dbg_mmio(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: mmio: "fmt, ##args)
+
+#define gvt_dbg_dpy(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: dpy: "fmt, ##args)
+
+#define gvt_dbg_el(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: el: "fmt, ##args)
+
+#define gvt_dbg_sched(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: sched: "fmt, ##args)
+
+#define gvt_dbg_render(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: render: "fmt, ##args)
+
+#define gvt_dbg_cmd(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: cmd: "fmt, ##args)
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
new file mode 100644
index 000000000000..c0c884aeb30e
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -0,0 +1,330 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+static int get_edp_pipe(struct intel_vgpu *vgpu)
+{
+        u32 data = vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP);
+        int pipe = -1;
+
+        switch (data & TRANS_DDI_EDP_INPUT_MASK) {
+        case TRANS_DDI_EDP_INPUT_A_ON:
+        case TRANS_DDI_EDP_INPUT_A_ONOFF:
+                pipe = PIPE_A;
+                break;
+        case TRANS_DDI_EDP_INPUT_B_ONOFF:
+                pipe = PIPE_B;
+                break;
+        case TRANS_DDI_EDP_INPUT_C_ONOFF:
+                pipe = PIPE_C;
+                break;
+        }
+        return pipe;
+}
+
+static int edp_pipe_is_enabled(struct intel_vgpu *vgpu)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+        if (!(vgpu_vreg(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE))
+                return 0;
+
+        if (!(vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP) & TRANS_DDI_FUNC_ENABLE))
+                return 0;
+        return 1;
+}
+
+static int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+        if (WARN_ON(pipe < PIPE_A || pipe >= I915_MAX_PIPES))
+                return -EINVAL;
+
+        if (vgpu_vreg(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE)
+                return 1;
+
+        if (edp_pipe_is_enabled(vgpu) &&
+                        get_edp_pipe(vgpu) == pipe)
+                return 1;
+        return 0;
+}
+
+/* EDID with 1024x768 as its resolution */
+static unsigned char virtual_dp_monitor_edid[] = {
+        /*Header*/
+        0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
+        /* Vendor & Product Identification */
+        0x22, 0xf0, 0x54, 0x29, 0x00, 0x00, 0x00, 0x00, 0x04, 0x17,
+        /* Version & Revision */
+        0x01, 0x04,
+        /* Basic Display Parameters & Features */
+        0xa5, 0x34, 0x20, 0x78, 0x23,
+        /* Color Characteristics */
+        0xfc, 0x81, 0xa4, 0x55, 0x4d, 0x9d, 0x25, 0x12, 0x50, 0x54,
+        /* Established Timings: maximum resolution is 1024x768 */
+        0x21, 0x08, 0x00,
+        /* Standard Timings. All invalid */
+        0x00, 0xc0, 0x00, 0xc0, 0x00, 0x40, 0x00, 0x80, 0x00, 0x00,
+        0x00, 0x40, 0x00, 0x00, 0x00, 0x01,
+        /* 18 Byte Data Blocks 1: invalid */
+        0x00, 0x00, 0x80, 0xa0, 0x70, 0xb0,
+        0x23, 0x40, 0x30, 0x20, 0x36, 0x00, 0x06, 0x44, 0x21, 0x00, 0x00, 0x1a,
+        /* 18 Byte Data Blocks 2: invalid */
+        0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x3c, 0x18, 0x50, 0x11, 0x00, 0x0a,
+        0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+        /* 18 Byte Data Blocks 3: invalid */
+        0x00, 0x00, 0x00, 0xfc, 0x00, 0x48,
+        0x50, 0x20, 0x5a, 0x52, 0x32, 0x34, 0x34, 0x30, 0x77, 0x0a, 0x20, 0x20,
+        /* 18 Byte Data Blocks 4: invalid */
+        0x00, 0x00, 0x00, 0xff, 0x00, 0x43, 0x4e, 0x34, 0x33, 0x30, 0x34, 0x30,
+        0x44, 0x58, 0x51, 0x0a, 0x20, 0x20,
+        /* Extension Block Count */
+        0x00,
+        /* Checksum */
+        0xef,
+};
+
+#define DPCD_HEADER_SIZE        0xb
+
+static u8 dpcd_fix_data[DPCD_HEADER_SIZE] = {
+        0x11, 0x0a, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTB_HOTPLUG_CPT |
+                        SDE_PORTC_HOTPLUG_CPT |
+                        SDE_PORTD_HOTPLUG_CPT);
+
+        if (IS_SKYLAKE(dev_priv))
+                vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT |
+                                SDE_PORTE_HOTPLUG_SPT);
+
+        if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B))
+                vgpu_vreg(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT;
+
+        if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C))
+                vgpu_vreg(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT;
+
+        if (intel_vgpu_has_monitor_on_port(vgpu, PORT_D))
+                vgpu_vreg(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT;
+
+        if (IS_SKYLAKE(dev_priv) &&
+                        intel_vgpu_has_monitor_on_port(vgpu, PORT_E)) {
+                vgpu_vreg(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT;
+        }
+
+        if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) {
+                if (IS_BROADWELL(dev_priv))
+                        vgpu_vreg(vgpu, GEN8_DE_PORT_ISR) |=
+                                GEN8_PORT_DP_A_HOTPLUG;
+                else
+                        vgpu_vreg(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT;
+        }
+}
+
+static void clean_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num)
+{
+        struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+
+        kfree(port->edid);
+        port->edid = NULL;
+
+        kfree(port->dpcd);
+        port->dpcd = NULL;
+}
+
+static int setup_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num,
+                int type)
+{
+        struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+
+        port->edid = kzalloc(sizeof(*(port->edid)), GFP_KERNEL);
+        if (!port->edid)
+                return -ENOMEM;
+
+        port->dpcd = kzalloc(sizeof(*(port->dpcd)), GFP_KERNEL);
+        if (!port->dpcd) {
+                kfree(port->edid);
+                return -ENOMEM;
+        }
+
+        memcpy(port->edid->edid_block, virtual_dp_monitor_edid,
+                        EDID_SIZE);
+        port->edid->data_valid = true;
+
+        memcpy(port->dpcd->data, dpcd_fix_data, DPCD_HEADER_SIZE);
+        port->dpcd->data_valid = true;
+        port->dpcd->data[DPCD_SINK_COUNT] = 0x1;
+        port->type = type;
+
+        emulate_monitor_status_change(vgpu);
+        return 0;
+}
+
+/**
+ * intel_gvt_check_vblank_emulation - check if vblank emulation timer should
+ * be turned on/off when a virtual pipe is enabled/disabled.
+ * @gvt: a GVT device
+ *
+ * This function is used to turn on/off vblank timer according to currently
+ * enabled/disabled virtual pipes.
+ *
+ */
+void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt)
+{
+        struct intel_gvt_irq *irq = &gvt->irq;
+        struct intel_vgpu *vgpu;
+        bool have_enabled_pipe = false;
+        int pipe, id;
+
+        if (WARN_ON(!mutex_is_locked(&gvt->lock)))
+                return;
+
+        hrtimer_cancel(&irq->vblank_timer.timer);
+
+        for_each_active_vgpu(gvt, vgpu, id) {
+                for (pipe = 0; pipe < I915_MAX_PIPES; pipe++) {
+                        have_enabled_pipe =
+                                pipe_is_enabled(vgpu, pipe);
+                        if (have_enabled_pipe)
+                                break;
+                }
+        }
+
+        if (have_enabled_pipe)
+                hrtimer_start(&irq->vblank_timer.timer,
+                        ktime_add_ns(ktime_get(), irq->vblank_timer.period),
+                        HRTIMER_MODE_ABS);
+}
+
+static void emulate_vblank_on_pipe(struct intel_vgpu *vgpu, int pipe)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct intel_vgpu_irq *irq = &vgpu->irq;
+        int vblank_event[] = {
+                [PIPE_A] = PIPE_A_VBLANK,
+                [PIPE_B] = PIPE_B_VBLANK,
+                [PIPE_C] = PIPE_C_VBLANK,
+        };
+        int event;
+
+        if (pipe < PIPE_A || pipe > PIPE_C)
+                return;
+
+        for_each_set_bit(event, irq->flip_done_event[pipe],
+                        INTEL_GVT_EVENT_MAX) {
+                clear_bit(event, irq->flip_done_event[pipe]);
+                if (!pipe_is_enabled(vgpu, pipe))
+                        continue;
+
+                vgpu_vreg(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++;
+                intel_vgpu_trigger_virtual_event(vgpu, event);
+        }
+
+        if (pipe_is_enabled(vgpu, pipe)) {
+                vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(pipe))++;
+                intel_vgpu_trigger_virtual_event(vgpu, vblank_event[pipe]);
+        }
+}
+
+static void emulate_vblank(struct intel_vgpu *vgpu)
+{
+        int pipe;
+
+        for_each_pipe(vgpu->gvt->dev_priv, pipe)
+                emulate_vblank_on_pipe(vgpu, pipe);
+}
+
+/**
+ * intel_gvt_emulate_vblank - trigger vblank events for vGPUs on GVT device
+ * @gvt: a GVT device
+ *
+ * This function is used to trigger vblank interrupts for vGPUs on GVT device
+ *
+ */
+void intel_gvt_emulate_vblank(struct intel_gvt *gvt)
+{
+        struct intel_vgpu *vgpu;
+        int id;
+
+        if (WARN_ON(!mutex_is_locked(&gvt->lock)))
+                return;
+
+        for_each_active_vgpu(gvt, vgpu, id)
+                emulate_vblank(vgpu);
+}
+
+/**
+ * intel_vgpu_clean_display - clean vGPU virtual display emulation
+ * @vgpu: a vGPU
+ *
+ * This function is used to clean vGPU virtual display emulation stuffs
+ *
+ */
+void intel_vgpu_clean_display(struct intel_vgpu *vgpu)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+        if (IS_SKYLAKE(dev_priv))
+                clean_virtual_dp_monitor(vgpu, PORT_D);
+        else
+                clean_virtual_dp_monitor(vgpu, PORT_B);
+}
+
+/**
+ * intel_vgpu_init_display- initialize vGPU virtual display emulation
+ * @vgpu: a vGPU
+ *
+ * This function is used to initialize vGPU virtual display emulation stuffs
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_init_display(struct intel_vgpu *vgpu)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+        intel_vgpu_init_i2c_edid(vgpu);
+
+        if (IS_SKYLAKE(dev_priv))
+                return setup_virtual_dp_monitor(vgpu, PORT_D, GVT_DP_D);
+        else
+                return setup_virtual_dp_monitor(vgpu, PORT_B, GVT_DP_B);
+}
diff --git a/drivers/gpu/drm/i915/gvt/display.h b/drivers/gpu/drm/i915/gvt/display.h
new file mode 100644
index 000000000000..7a60cb848268
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/display.h
@@ -0,0 +1,163 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef _GVT_DISPLAY_H_
+#define _GVT_DISPLAY_H_
+
+#define SBI_REG_MAX     20
+#define DPCD_SIZE       0x700
+
+#define intel_vgpu_port(vgpu, port) \
+        (&(vgpu->display.ports[port]))
+
+#define intel_vgpu_has_monitor_on_port(vgpu, port) \
+        (intel_vgpu_port(vgpu, port)->edid && \
+                intel_vgpu_port(vgpu, port)->edid->data_valid)
+
+#define intel_vgpu_port_is_dp(vgpu, port) \
+        ((intel_vgpu_port(vgpu, port)->type == GVT_DP_A) || \
+        (intel_vgpu_port(vgpu, port)->type == GVT_DP_B) || \
+        (intel_vgpu_port(vgpu, port)->type == GVT_DP_C) || \
+        (intel_vgpu_port(vgpu, port)->type == GVT_DP_D))
+
+#define INTEL_GVT_MAX_UEVENT_VARS       3
+
+/* DPCD start */
+#define DPCD_SIZE       0x700
+
+/* DPCD */
+#define DP_SET_POWER            0x600
+#define DP_SET_POWER_D0         0x1
+#define AUX_NATIVE_WRITE        0x8
+#define AUX_NATIVE_READ         0x9
+
+#define AUX_NATIVE_REPLY_MASK   (0x3 << 4)
+#define AUX_NATIVE_REPLY_ACK    (0x0 << 4)
+#define AUX_NATIVE_REPLY_NAK    (0x1 << 4)
+#define AUX_NATIVE_REPLY_DEFER  (0x2 << 4)
+
+#define AUX_BURST_SIZE          16
+
+/* DPCD addresses */
+#define DPCD_REV                        0x000
+#define DPCD_MAX_LINK_RATE              0x001
+#define DPCD_MAX_LANE_COUNT             0x002
+
+#define DPCD_TRAINING_PATTERN_SET       0x102
+#define DPCD_SINK_COUNT                 0x200
+#define DPCD_LANE0_1_STATUS             0x202
+#define DPCD_LANE2_3_STATUS             0x203
+#define DPCD_LANE_ALIGN_STATUS_UPDATED  0x204
+#define DPCD_SINK_STATUS                0x205
+
+/* link training */
+#define DPCD_TRAINING_PATTERN_SET_MASK  0x03
+#define DPCD_LINK_TRAINING_DISABLED     0x00
+#define DPCD_TRAINING_PATTERN_1         0x01
+#define DPCD_TRAINING_PATTERN_2         0x02
+
+#define DPCD_CP_READY_MASK              (1 << 6)
+
+/* lane status */
+#define DPCD_LANES_CR_DONE              0x11
+#define DPCD_LANES_EQ_DONE              0x22
+#define DPCD_SYMBOL_LOCKED              0x44
+
+#define DPCD_INTERLANE_ALIGN_DONE       0x01
+
+#define DPCD_SINK_IN_SYNC               0x03
+/* DPCD end */
+
+#define SBI_RESPONSE_MASK               0x3
+#define SBI_RESPONSE_SHIFT              0x1
+#define SBI_STAT_MASK                   0x1
+#define SBI_STAT_SHIFT                  0x0
+#define SBI_OPCODE_SHIFT                8
+#define SBI_OPCODE_MASK                 (0xff << SBI_OPCODE_SHIFT)
+#define SBI_CMD_IORD                    2
+#define SBI_CMD_IOWR                    3
+#define SBI_CMD_CRRD                    6
+#define SBI_CMD_CRWR                    7
+#define SBI_ADDR_OFFSET_SHIFT           16
+#define SBI_ADDR_OFFSET_MASK            (0xffff << SBI_ADDR_OFFSET_SHIFT)
+
+struct intel_vgpu_sbi_register {
+        unsigned int offset;
+        u32 value;
+};
+
+struct intel_vgpu_sbi {
+        int number;
+        struct intel_vgpu_sbi_register registers[SBI_REG_MAX];
+};
+
+enum intel_gvt_plane_type {
+        PRIMARY_PLANE = 0,
+        CURSOR_PLANE,
+        SPRITE_PLANE,
+        MAX_PLANE
+};
+
+struct intel_vgpu_dpcd_data {
+        bool data_valid;
+        u8 data[DPCD_SIZE];
+};
+
+enum intel_vgpu_port_type {
+        GVT_CRT = 0,
+        GVT_DP_A,
+        GVT_DP_B,
+        GVT_DP_C,
+        GVT_DP_D,
+        GVT_HDMI_B,
+        GVT_HDMI_C,
+        GVT_HDMI_D,
+        GVT_PORT_MAX
+};
+
+struct intel_vgpu_port {
+        /* per display EDID information */
+        struct intel_vgpu_edid_data *edid;
+        /* per display DPCD information */
+        struct intel_vgpu_dpcd_data *dpcd;
+        int type;
+};
+
+void intel_gvt_emulate_vblank(struct intel_gvt *gvt);
+void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt);
+
+int intel_vgpu_init_display(struct intel_vgpu *vgpu);
+void intel_vgpu_clean_display(struct intel_vgpu *vgpu);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
new file mode 100644
index 000000000000..7e1da1c563ca
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -0,0 +1,532 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define GMBUS1_TOTAL_BYTES_SHIFT 16
+#define GMBUS1_TOTAL_BYTES_MASK 0x1ff
+#define gmbus1_total_byte_count(v) (((v) >> \
+        GMBUS1_TOTAL_BYTES_SHIFT) & GMBUS1_TOTAL_BYTES_MASK)
+#define gmbus1_slave_addr(v) (((v) & 0xff) >> 1)
+#define gmbus1_slave_index(v) (((v) >> 8) & 0xff)
+#define gmbus1_bus_cycle(v) (((v) >> 25) & 0x7)
+
+/* GMBUS0 bits definitions */
+#define _GMBUS_PIN_SEL_MASK     (0x7)
+
+static unsigned char edid_get_byte(struct intel_vgpu *vgpu)
+{
+        struct intel_vgpu_i2c_edid *edid = &vgpu->display.i2c_edid;
+        unsigned char chr = 0;
+
+        if (edid->state == I2C_NOT_SPECIFIED || !edid->slave_selected) {
+                gvt_err("Driver tries to read EDID without proper sequence!\n");
+                return 0;
+        }
+        if (edid->current_edid_read >= EDID_SIZE) {
+                gvt_err("edid_get_byte() exceeds the size of EDID!\n");
+                return 0;
+        }
+
+        if (!edid->edid_available) {
+                gvt_err("Reading EDID but EDID is not available!\n");
+                return 0;
+        }
+
+        if (intel_vgpu_has_monitor_on_port(vgpu, edid->port)) {
+                struct intel_vgpu_edid_data *edid_data =
+                        intel_vgpu_port(vgpu, edid->port)->edid;
+
+                chr = edid_data->edid_block[edid->current_edid_read];
+                edid->current_edid_read++;
+        } else {
+                gvt_err("No EDID available during the reading?\n");
+        }
+        return chr;
+}
+
+static inline int get_port_from_gmbus0(u32 gmbus0)
+{
+        int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK;
+        int port = -EINVAL;
+
+        if (port_select == 2)
+                port = PORT_E;
+        else if (port_select == 4)
+                port = PORT_C;
+        else if (port_select == 5)
+                port = PORT_B;
+        else if (port_select == 6)
+                port = PORT_D;
+        return port;
+}
+
+static void reset_gmbus_controller(struct intel_vgpu *vgpu)
+{
+        vgpu_vreg(vgpu, PCH_GMBUS2) = GMBUS_HW_RDY;
+        if (!vgpu->display.i2c_edid.edid_available)
+                vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
+        vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE;
+}
+
+/* GMBUS0 */
+static int gmbus0_mmio_write(struct intel_vgpu *vgpu,
+                        unsigned int offset, void *p_data, unsigned int bytes)
+{
+        int port, pin_select;
+
+        memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes);
+
+        pin_select = vgpu_vreg(vgpu, offset) & _GMBUS_PIN_SEL_MASK;
+
+        intel_vgpu_init_i2c_edid(vgpu);
+
+        if (pin_select == 0)
+                return 0;
+
+        port = get_port_from_gmbus0(pin_select);
+        if (WARN_ON(port < 0))
+                return 0;
+
+        vgpu->display.i2c_edid.state = I2C_GMBUS;
+        vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE;
+
+        vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
+        vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY | GMBUS_HW_WAIT_PHASE;
+
+        if (intel_vgpu_has_monitor_on_port(vgpu, port) &&
+                        !intel_vgpu_port_is_dp(vgpu, port)) {
+                vgpu->display.i2c_edid.port = port;
+                vgpu->display.i2c_edid.edid_available = true;
+                vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_SATOER;
+        } else
+                vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
+        return 0;
+}
+
+static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid;
+        u32 slave_addr;
+        u32 wvalue = *(u32 *)p_data;
+
+        if (vgpu_vreg(vgpu, offset) & GMBUS_SW_CLR_INT) {
+                if (!(wvalue & GMBUS_SW_CLR_INT)) {
+                        vgpu_vreg(vgpu, offset) &= ~GMBUS_SW_CLR_INT;
+                        reset_gmbus_controller(vgpu);
+                }
+                /*
+                 * TODO: "This bit is cleared to zero when an event
+                 * causes the HW_RDY bit transition to occur "
+                 */
+        } else {
+                /*
+                 * per bspec setting this bit can cause:
+                 * 1) INT status bit cleared
+                 * 2) HW_RDY bit asserted
+                 */
+                if (wvalue & GMBUS_SW_CLR_INT) {
+                        vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_INT;
+                        vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY;
+                }
+
+                /* For virtualization, we suppose that HW is always ready,
+                 * so GMBUS_SW_RDY should always be cleared
+                 */
+                if (wvalue & GMBUS_SW_RDY)
+                        wvalue &= ~GMBUS_SW_RDY;
+
+                i2c_edid->gmbus.total_byte_count =
+                        gmbus1_total_byte_count(wvalue);
+                slave_addr = gmbus1_slave_addr(wvalue);
+
+                /* vgpu gmbus only support EDID */
+                if (slave_addr == EDID_ADDR) {
+                        i2c_edid->slave_selected = true;
+                } else if (slave_addr != 0) {
+                        gvt_dbg_dpy(
+                                "vgpu%d: unsupported gmbus slave addr(0x%x)\n"
+                                "       gmbus operations will be ignored.\n",
+                                        vgpu->id, slave_addr);
+                }
+
+                if (wvalue & GMBUS_CYCLE_INDEX)
+                        i2c_edid->current_edid_read =
+                                gmbus1_slave_index(wvalue);
+
+                i2c_edid->gmbus.cycle_type = gmbus1_bus_cycle(wvalue);
+                switch (gmbus1_bus_cycle(wvalue)) {
+                case GMBUS_NOCYCLE:
+                        break;
+                case GMBUS_STOP:
+                        /* From spec:
+                         * This can only cause a STOP to be generated
+                         * if a GMBUS cycle is generated, the GMBUS is
+                         * currently in a data/wait/idle phase, or it is in a
+                         * WAIT phase
+                         */
+                        if (gmbus1_bus_cycle(vgpu_vreg(vgpu, offset))
+                                != GMBUS_NOCYCLE) {
+                                intel_vgpu_init_i2c_edid(vgpu);
+                                /* After the 'stop' cycle, hw state would become
+                                 * 'stop phase' and then 'idle phase' after a
+                                 * few milliseconds. In emulation, we just set
+                                 * it as 'idle phase' ('stop phase' is not
+                                 * visible in gmbus interface)
+                                 */
+                                i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE;
+                                vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
+                        }
+                        break;
+                case NIDX_NS_W:
+                case IDX_NS_W:
+                case NIDX_STOP:
+                case IDX_STOP:
+                        /* From hw spec the GMBUS phase
+                         * transition like this:
+                         * START (-->INDEX) -->DATA
+                         */
+                        i2c_edid->gmbus.phase = GMBUS_DATA_PHASE;
+                        vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_ACTIVE;
+                        break;
+                default:
+                        gvt_err("Unknown/reserved GMBUS cycle detected!\n");
+                        break;
+                }
+                /*
+                 * From hw spec the WAIT state will be
+                 * cleared:
+                 * (1) in a new GMBUS cycle
+                 * (2) by generating a stop
+                 */
+                vgpu_vreg(vgpu, offset) = wvalue;
+        }
+        return 0;
+}
+
+static int gmbus3_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+        void *p_data, unsigned int bytes)
+{
+        WARN_ON(1);
+        return 0;
+}
+
+static int gmbus3_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        int i;
+        unsigned char byte_data;
+        struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid;
+        int byte_left = i2c_edid->gmbus.total_byte_count -
+                                i2c_edid->current_edid_read;
+        int byte_count = byte_left;
+        u32 reg_data = 0;
+
+        /* Data can only be recevied if previous settings correct */
+        if (vgpu_vreg(vgpu, PCH_GMBUS1) & GMBUS_SLAVE_READ) {
+                if (byte_left <= 0) {
+                        memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+                        return 0;
+                }
+
+                if (byte_count > 4)
+                        byte_count = 4;
+                for (i = 0; i < byte_count; i++) {
+                        byte_data = edid_get_byte(vgpu);
+                        reg_data |= (byte_data << (i << 3));
+                }
+
+                memcpy(&vgpu_vreg(vgpu, offset), &reg_data, byte_count);
+                memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+
+                if (byte_left <= 4) {
+                        switch (i2c_edid->gmbus.cycle_type) {
+                        case NIDX_STOP:
+                        case IDX_STOP:
+                                i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE;
+                                break;
+                        case NIDX_NS_W:
+                        case IDX_NS_W:
+                        default:
+                                i2c_edid->gmbus.phase = GMBUS_WAIT_PHASE;
+                                break;
+                        }
+                        intel_vgpu_init_i2c_edid(vgpu);
+                }
+                /*
+                 * Read GMBUS3 during send operation,
+                 * return the latest written value
+                 */
+        } else {
+                memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+                gvt_err("vgpu%d: warning: gmbus3 read with nothing returned\n",
+                                vgpu->id);
+        }
+        return 0;
+}
+
+static int gmbus2_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 value = vgpu_vreg(vgpu, offset);
+
+        if (!(vgpu_vreg(vgpu, offset) & GMBUS_INUSE))
+                vgpu_vreg(vgpu, offset) |= GMBUS_INUSE;
+        memcpy(p_data, (void *)&value, bytes);
+        return 0;
+}
+
+static int gmbus2_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 wvalue = *(u32 *)p_data;
+
+        if (wvalue & GMBUS_INUSE)
+                vgpu_vreg(vgpu, offset) &= ~GMBUS_INUSE;
+        /* All other bits are read-only */
+        return 0;
+}
+
+/**
+ * intel_gvt_i2c_handle_gmbus_read - emulate gmbus register mmio read
+ * @vgpu: a vGPU
+ *
+ * This function is used to emulate gmbus register mmio read
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_gvt_i2c_handle_gmbus_read(struct intel_vgpu *vgpu,
+        unsigned int offset, void *p_data, unsigned int bytes)
+{
+        if (WARN_ON(bytes > 8 && (offset & (bytes - 1))))
+                return -EINVAL;
+
+        if (offset == i915_mmio_reg_offset(PCH_GMBUS2))
+                return gmbus2_mmio_read(vgpu, offset, p_data, bytes);
+        else if (offset == i915_mmio_reg_offset(PCH_GMBUS3))
+                return gmbus3_mmio_read(vgpu, offset, p_data, bytes);
+
+        memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+        return 0;
+}
+
+/**
+ * intel_gvt_i2c_handle_gmbus_write - emulate gmbus register mmio write
+ * @vgpu: a vGPU
+ *
+ * This function is used to emulate gmbus register mmio write
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_gvt_i2c_handle_gmbus_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        if (WARN_ON(bytes > 8 && (offset & (bytes - 1))))
+                return -EINVAL;
+
+        if (offset == i915_mmio_reg_offset(PCH_GMBUS0))
+                return gmbus0_mmio_write(vgpu, offset, p_data, bytes);
+        else if (offset == i915_mmio_reg_offset(PCH_GMBUS1))
+                return gmbus1_mmio_write(vgpu, offset, p_data, bytes);
+        else if (offset == i915_mmio_reg_offset(PCH_GMBUS2))
+                return gmbus2_mmio_write(vgpu, offset, p_data, bytes);
+        else if (offset == i915_mmio_reg_offset(PCH_GMBUS3))
+                return gmbus3_mmio_write(vgpu, offset, p_data, bytes);
+
+        memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes);
+        return 0;
+}
+
+enum {
+        AUX_CH_CTL = 0,
+        AUX_CH_DATA1,
+        AUX_CH_DATA2,
+        AUX_CH_DATA3,
+        AUX_CH_DATA4,
+        AUX_CH_DATA5
+};
+
+static inline int get_aux_ch_reg(unsigned int offset)
+{
+        int reg;
+
+        switch (offset & 0xff) {
+        case 0x10:
+                reg = AUX_CH_CTL;
+                break;
+        case 0x14:
+                reg = AUX_CH_DATA1;
+                break;
+        case 0x18:
+                reg = AUX_CH_DATA2;
+                break;
+        case 0x1c:
+                reg = AUX_CH_DATA3;
+                break;
+        case 0x20:
+                reg = AUX_CH_DATA4;
+                break;
+        case 0x24:
+                reg = AUX_CH_DATA5;
+                break;
+        default:
+                reg = -1;
+                break;
+        }
+        return reg;
+}
+
+#define AUX_CTL_MSG_LENGTH(reg) \
+        ((reg & DP_AUX_CH_CTL_MESSAGE_SIZE_MASK) >> \
+                DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT)
+
+/**
+ * intel_gvt_i2c_handle_aux_ch_write - emulate AUX channel register write
+ * @vgpu: a vGPU
+ *
+ * This function is used to emulate AUX channel register write
+ *
+ */
+void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu,
+                                int port_idx,
+                                unsigned int offset,
+                                void *p_data)
+{
+        struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid;
+        int msg_length, ret_msg_size;
+        int msg, addr, ctrl, op;
+        u32 value = *(u32 *)p_data;
+        int aux_data_for_write = 0;
+        int reg = get_aux_ch_reg(offset);
+
+        if (reg != AUX_CH_CTL) {
+                vgpu_vreg(vgpu, offset) = value;
+                return;
+        }
+
+        msg_length = AUX_CTL_MSG_LENGTH(value);
+        // check the msg in DATA register.
+        msg = vgpu_vreg(vgpu, offset + 4);
+        addr = (msg >> 8) & 0xffff;
+        ctrl = (msg >> 24) & 0xff;
+        op = ctrl >> 4;
+        if (!(value & DP_AUX_CH_CTL_SEND_BUSY)) {
+                /* The ctl write to clear some states */
+                return;
+        }
+
+        /* Always set the wanted value for vms. */
+        ret_msg_size = (((op & 0x1) == GVT_AUX_I2C_READ) ? 2 : 1);
+        vgpu_vreg(vgpu, offset) =
+                DP_AUX_CH_CTL_DONE |
+                ((ret_msg_size << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) &
+                DP_AUX_CH_CTL_MESSAGE_SIZE_MASK);
+
+        if (msg_length == 3) {
+                if (!(op & GVT_AUX_I2C_MOT)) {
+                        /* stop */
+                        intel_vgpu_init_i2c_edid(vgpu);
+                } else {
+                        /* start or restart */
+                        i2c_edid->aux_ch.i2c_over_aux_ch = true;
+                        i2c_edid->aux_ch.aux_ch_mot = true;
+                        if (addr == 0) {
+                                /* reset the address */
+                                intel_vgpu_init_i2c_edid(vgpu);
+                        } else if (addr == EDID_ADDR) {
+                                i2c_edid->state = I2C_AUX_CH;
+                                i2c_edid->port = port_idx;
+                                i2c_edid->slave_selected = true;
+                                if (intel_vgpu_has_monitor_on_port(vgpu,
+                                        port_idx) &&
+                                        intel_vgpu_port_is_dp(vgpu, port_idx))
+                                        i2c_edid->edid_available = true;
+                        }
+                }
+        } else if ((op & 0x1) == GVT_AUX_I2C_WRITE) {
+                /* TODO
+                 * We only support EDID reading from I2C_over_AUX. And
+                 * we do not expect the index mode to be used. Right now
+                 * the WRITE operation is ignored. It is good enough to
+                 * support the gfx driver to do EDID access.
+                 */
+        } else {
+                if (WARN_ON((op & 0x1) != GVT_AUX_I2C_READ))
+                        return;
+                if (WARN_ON(msg_length != 4))
+                        return;
+                if (i2c_edid->edid_available && i2c_edid->slave_selected) {
+                        unsigned char val = edid_get_byte(vgpu);
+
+                        aux_data_for_write = (val << 16);
+                }
+        }
+        /* write the return value in AUX_CH_DATA reg which includes:
+         * ACK of I2C_WRITE
+         * returned byte if it is READ
+         */
+
+        aux_data_for_write |= (GVT_AUX_I2C_REPLY_ACK & 0xff) << 24;
+        vgpu_vreg(vgpu, offset + 4) = aux_data_for_write;
+}
+
+/**
+ * intel_vgpu_init_i2c_edid - initialize vGPU i2c edid emulation
+ * @vgpu: a vGPU
+ *
+ * This function is used to initialize vGPU i2c edid emulation stuffs
+ *
+ */
+void intel_vgpu_init_i2c_edid(struct intel_vgpu *vgpu)
+{
+        struct intel_vgpu_i2c_edid *edid = &vgpu->display.i2c_edid;
+
+        edid->state = I2C_NOT_SPECIFIED;
+
+        edid->port = -1;
+        edid->slave_selected = false;
+        edid->edid_available = false;
+        edid->current_edid_read = 0;
+
+        memset(&edid->gmbus, 0, sizeof(struct intel_vgpu_i2c_gmbus));
+
+        edid->aux_ch.i2c_over_aux_ch = false;
+        edid->aux_ch.aux_ch_mot = false;
+}
diff --git a/drivers/gpu/drm/i915/gvt/edid.h b/drivers/gpu/drm/i915/gvt/edid.h
new file mode 100644
index 000000000000..de366b1d5196
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/edid.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef _GVT_EDID_H_
+#define _GVT_EDID_H_
+
+#define EDID_SIZE               128
+#define EDID_ADDR               0x50 /* Linux hvm EDID addr */
+
+#define GVT_AUX_NATIVE_WRITE                    0x8
+#define GVT_AUX_NATIVE_READ                     0x9
+#define GVT_AUX_I2C_WRITE                       0x0
+#define GVT_AUX_I2C_READ                        0x1
+#define GVT_AUX_I2C_STATUS                      0x2
+#define GVT_AUX_I2C_MOT                         0x4
+#define GVT_AUX_I2C_REPLY_ACK                   (0x0 << 6)
+
+struct intel_vgpu_edid_data {
+        bool data_valid;
+        unsigned char edid_block[EDID_SIZE];
+};
+
+enum gmbus_cycle_type {
+        GMBUS_NOCYCLE   = 0x0,
+        NIDX_NS_W       = 0x1,
+        IDX_NS_W        = 0x3,
+        GMBUS_STOP      = 0x4,
+        NIDX_STOP       = 0x5,
+        IDX_STOP        = 0x7
+};
+
+/*
+ * States of GMBUS
+ *
+ * GMBUS0-3 could be related to the EDID virtualization. Another two GMBUS
+ * registers, GMBUS4 (interrupt mask) and GMBUS5 (2 byte indes register), are
+ * not considered here. Below describes the usage of GMBUS registers that are
+ * cared by the EDID virtualization
+ *
+ * GMBUS0:
+ *      R/W
+ *      port selection. value of bit0 - bit2 corresponds to the GPIO registers.
+ *
+ * GMBUS1:
+ *      R/W Protect
+ *      Command and Status.
+ *      bit0 is the direction bit: 1 is read; 0 is write.
+ *      bit1 - bit7 is slave 7-bit address.
+ *      bit16 - bit24 total byte count (ignore?)
+ *
+ * GMBUS2:
+ *      Most of bits are read only except bit 15 (IN_USE)
+ *      Status register
+ *      bit0 - bit8 current byte count
+ *      bit 11: hardware ready;
+ *
+ * GMBUS3:
+ *      Read/Write
+ *      Data for transfer
+ */
+
+/* From hw specs, Other phases like START, ADDRESS, INDEX
+ * are invisible to GMBUS MMIO interface. So no definitions
+ * in below enum types
+ */
+enum gvt_gmbus_phase {
+        GMBUS_IDLE_PHASE = 0,
+        GMBUS_DATA_PHASE,
+        GMBUS_WAIT_PHASE,
+        //GMBUS_STOP_PHASE,
+        GMBUS_MAX_PHASE
+};
+
+struct intel_vgpu_i2c_gmbus {
+        unsigned int total_byte_count; /* from GMBUS1 */
+        enum gmbus_cycle_type cycle_type;
+        enum gvt_gmbus_phase phase;
+};
+
+struct intel_vgpu_i2c_aux_ch {
+        bool i2c_over_aux_ch;
+        bool aux_ch_mot;
+};
+
+enum i2c_state {
+        I2C_NOT_SPECIFIED = 0,
+        I2C_GMBUS = 1,
+        I2C_AUX_CH = 2
+};
+
+/* I2C sequences cannot interleave.
+ * GMBUS and AUX_CH sequences cannot interleave.
+ */
+struct intel_vgpu_i2c_edid {
+        enum i2c_state state;
+
+        unsigned int port;
+        bool slave_selected;
+        bool edid_available;
+        unsigned int current_edid_read;
+
+        struct intel_vgpu_i2c_gmbus gmbus;
+        struct intel_vgpu_i2c_aux_ch aux_ch;
+};
+
+void intel_vgpu_init_i2c_edid(struct intel_vgpu *vgpu);
+
+int intel_gvt_i2c_handle_gmbus_read(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes);
+
+int intel_gvt_i2c_handle_gmbus_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes);
+
+void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu,
+                int port_idx,
+                unsigned int offset,
+                void *p_data);
+
+#endif /*_GVT_EDID_H_*/
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
new file mode 100644
index 000000000000..c1f6019d8895
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -0,0 +1,860 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define _EL_OFFSET_STATUS       0x234
+#define _EL_OFFSET_STATUS_BUF   0x370
+#define _EL_OFFSET_STATUS_PTR   0x3A0
+
+#define execlist_ring_mmio(gvt, ring_id, offset) \
+        (gvt->dev_priv->engine[ring_id]->mmio_base + (offset))
+
+#define valid_context(ctx) ((ctx)->valid)
+#define same_context(a, b) (((a)->context_id == (b)->context_id) && \
+                ((a)->lrca == (b)->lrca))
+
+static int context_switch_events[] = {
+        [RCS] = RCS_AS_CONTEXT_SWITCH,
+        [BCS] = BCS_AS_CONTEXT_SWITCH,
+        [VCS] = VCS_AS_CONTEXT_SWITCH,
+        [VCS2] = VCS2_AS_CONTEXT_SWITCH,
+        [VECS] = VECS_AS_CONTEXT_SWITCH,
+};
+
+static int ring_id_to_context_switch_event(int ring_id)
+{
+        if (WARN_ON(ring_id < RCS && ring_id >
+                                ARRAY_SIZE(context_switch_events)))
+                return -EINVAL;
+
+        return context_switch_events[ring_id];
+}
+
+static void switch_virtual_execlist_slot(struct intel_vgpu_execlist *execlist)
+{
+        gvt_dbg_el("[before] running slot %d/context %x pending slot %d\n",
+                        execlist->running_slot ?
+                        execlist->running_slot->index : -1,
+                        execlist->running_context ?
+                        execlist->running_context->context_id : 0,
+                        execlist->pending_slot ?
+                        execlist->pending_slot->index : -1);
+
+        execlist->running_slot = execlist->pending_slot;
+        execlist->pending_slot = NULL;
+        execlist->running_context = execlist->running_context ?
+                &execlist->running_slot->ctx[0] : NULL;
+
+        gvt_dbg_el("[after] running slot %d/context %x pending slot %d\n",
+                        execlist->running_slot ?
+                        execlist->running_slot->index : -1,
+                        execlist->running_context ?
+                        execlist->running_context->context_id : 0,
+                        execlist->pending_slot ?
+                        execlist->pending_slot->index : -1);
+}
+
+static void emulate_execlist_status(struct intel_vgpu_execlist *execlist)
+{
+        struct intel_vgpu_execlist_slot *running = execlist->running_slot;
+        struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
+        struct execlist_ctx_descriptor_format *desc = execlist->running_context;
+        struct intel_vgpu *vgpu = execlist->vgpu;
+        struct execlist_status_format status;
+        int ring_id = execlist->ring_id;
+        u32 status_reg = execlist_ring_mmio(vgpu->gvt,
+                        ring_id, _EL_OFFSET_STATUS);
+
+        status.ldw = vgpu_vreg(vgpu, status_reg);
+        status.udw = vgpu_vreg(vgpu, status_reg + 4);
+
+        if (running) {
+                status.current_execlist_pointer = !!running->index;
+                status.execlist_write_pointer = !!!running->index;
+                status.execlist_0_active = status.execlist_0_valid =
+                        !!!(running->index);
+                status.execlist_1_active = status.execlist_1_valid =
+                        !!(running->index);
+        } else {
+                status.context_id = 0;
+                status.execlist_0_active = status.execlist_0_valid = 0;
+                status.execlist_1_active = status.execlist_1_valid = 0;
+        }
+
+        status.context_id = desc ? desc->context_id : 0;
+        status.execlist_queue_full = !!(pending);
+
+        vgpu_vreg(vgpu, status_reg) = status.ldw;
+        vgpu_vreg(vgpu, status_reg + 4) = status.udw;
+
+        gvt_dbg_el("vgpu%d: status reg offset %x ldw %x udw %x\n",
+                vgpu->id, status_reg, status.ldw, status.udw);
+}
+
+static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
+                struct execlist_context_status_format *status,
+                bool trigger_interrupt_later)
+{
+        struct intel_vgpu *vgpu = execlist->vgpu;
+        int ring_id = execlist->ring_id;
+        struct execlist_context_status_pointer_format ctx_status_ptr;
+        u32 write_pointer;
+        u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset;
+
+        ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+                        _EL_OFFSET_STATUS_PTR);
+        ctx_status_buf_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+                        _EL_OFFSET_STATUS_BUF);
+
+        ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
+
+        write_pointer = ctx_status_ptr.write_ptr;
+
+        if (write_pointer == 0x7)
+                write_pointer = 0;
+        else {
+                ++write_pointer;
+                write_pointer %= 0x6;
+        }
+
+        offset = ctx_status_buf_reg + write_pointer * 8;
+
+        vgpu_vreg(vgpu, offset) = status->ldw;
+        vgpu_vreg(vgpu, offset + 4) = status->udw;
+
+        ctx_status_ptr.write_ptr = write_pointer;
+        vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
+
+        gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n",
+                vgpu->id, write_pointer, offset, status->ldw, status->udw);
+
+        if (trigger_interrupt_later)
+                return;
+
+        intel_vgpu_trigger_virtual_event(vgpu,
+                        ring_id_to_context_switch_event(execlist->ring_id));
+}
+
+static int emulate_execlist_ctx_schedule_out(
+                struct intel_vgpu_execlist *execlist,
+                struct execlist_ctx_descriptor_format *ctx)
+{
+        struct intel_vgpu_execlist_slot *running = execlist->running_slot;
+        struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
+        struct execlist_ctx_descriptor_format *ctx0 = &running->ctx[0];
+        struct execlist_ctx_descriptor_format *ctx1 = &running->ctx[1];
+        struct execlist_context_status_format status;
+
+        memset(&status, 0, sizeof(status));
+
+        gvt_dbg_el("schedule out context id %x\n", ctx->context_id);
+
+        if (WARN_ON(!same_context(ctx, execlist->running_context))) {
+                gvt_err("schedule out context is not running context,"
+                                "ctx id %x running ctx id %x\n",
+                                ctx->context_id,
+                                execlist->running_context->context_id);
+                return -EINVAL;
+        }
+
+        /* ctx1 is valid, ctx0/ctx is scheduled-out -> element switch */
+        if (valid_context(ctx1) && same_context(ctx0, ctx)) {
+                gvt_dbg_el("ctx 1 valid, ctx/ctx 0 is scheduled-out\n");
+
+                execlist->running_context = ctx1;
+
+                emulate_execlist_status(execlist);
+
+                status.context_complete = status.element_switch = 1;
+                status.context_id = ctx->context_id;
+
+                emulate_csb_update(execlist, &status, false);
+                /*
+                 * ctx1 is not valid, ctx == ctx0
+                 * ctx1 is valid, ctx1 == ctx
+                 *      --> last element is finished
+                 * emulate:
+                 *      active-to-idle if there is *no* pending execlist
+                 *      context-complete if there *is* pending execlist
+                 */
+        } else if ((!valid_context(ctx1) && same_context(ctx0, ctx))
+                        || (valid_context(ctx1) && same_context(ctx1, ctx))) {
+                gvt_dbg_el("need to switch virtual execlist slot\n");
+
+                switch_virtual_execlist_slot(execlist);
+
+                emulate_execlist_status(execlist);
+
+                status.context_complete = status.active_to_idle = 1;
+                status.context_id = ctx->context_id;
+
+                if (!pending) {
+                        emulate_csb_update(execlist, &status, false);
+                } else {
+                        emulate_csb_update(execlist, &status, true);
+
+                        memset(&status, 0, sizeof(status));
+
+                        status.idle_to_active = 1;
+                        status.context_id = 0;
+
+                        emulate_csb_update(execlist, &status, false);
+                }
+        } else {
+                WARN_ON(1);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static struct intel_vgpu_execlist_slot *get_next_execlist_slot(
+                struct intel_vgpu_execlist *execlist)
+{
+        struct intel_vgpu *vgpu = execlist->vgpu;
+        int ring_id = execlist->ring_id;
+        u32 status_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+                        _EL_OFFSET_STATUS);
+        struct execlist_status_format status;
+
+        status.ldw = vgpu_vreg(vgpu, status_reg);
+        status.udw = vgpu_vreg(vgpu, status_reg + 4);
+
+        if (status.execlist_queue_full) {
+                gvt_err("virtual execlist slots are full\n");
+                return NULL;
+        }
+
+        return &execlist->slot[status.execlist_write_pointer];
+}
+
+static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist,
+                struct execlist_ctx_descriptor_format ctx[2])
+{
+        struct intel_vgpu_execlist_slot *running = execlist->running_slot;
+        struct intel_vgpu_execlist_slot *slot =
+                get_next_execlist_slot(execlist);
+
+        struct execlist_ctx_descriptor_format *ctx0, *ctx1;
+        struct execlist_context_status_format status;
+
+        gvt_dbg_el("emulate schedule-in\n");
+
+        if (!slot) {
+                gvt_err("no available execlist slot\n");
+                return -EINVAL;
+        }
+
+        memset(&status, 0, sizeof(status));
+        memset(slot->ctx, 0, sizeof(slot->ctx));
+
+        slot->ctx[0] = ctx[0];
+        slot->ctx[1] = ctx[1];
+
+        gvt_dbg_el("alloc slot index %d ctx 0 %x ctx 1 %x\n",
+                        slot->index, ctx[0].context_id,
+                        ctx[1].context_id);
+
+        /*
+         * no running execlist, make this write bundle as running execlist
+         * -> idle-to-active
+         */
+        if (!running) {
+                gvt_dbg_el("no current running execlist\n");
+
+                execlist->running_slot = slot;
+                execlist->pending_slot = NULL;
+                execlist->running_context = &slot->ctx[0];
+
+                gvt_dbg_el("running slot index %d running context %x\n",
+                                execlist->running_slot->index,
+                                execlist->running_context->context_id);
+
+                emulate_execlist_status(execlist);
+
+                status.idle_to_active = 1;
+                status.context_id = 0;
+
+                emulate_csb_update(execlist, &status, false);
+                return 0;
+        }
+
+        ctx0 = &running->ctx[0];
+        ctx1 = &running->ctx[1];
+
+        gvt_dbg_el("current running slot index %d ctx 0 %x ctx 1 %x\n",
+                running->index, ctx0->context_id, ctx1->context_id);
+
+        /*
+         * already has an running execlist
+         *      a. running ctx1 is valid,
+         *         ctx0 is finished, and running ctx1 == new execlist ctx[0]
+         *      b. running ctx1 is not valid,
+         *         ctx0 == new execlist ctx[0]
+         * ----> lite-restore + preempted
+         */
+        if ((valid_context(ctx1) && same_context(ctx1, &slot->ctx[0]) &&
+                /* condition a */
+                (!same_context(ctx0, execlist->running_context))) ||
+                        (!valid_context(ctx1) &&
+                         same_context(ctx0, &slot->ctx[0]))) { /* condition b */
+                gvt_dbg_el("need to switch virtual execlist slot\n");
+
+                execlist->pending_slot = slot;
+                switch_virtual_execlist_slot(execlist);
+
+                emulate_execlist_status(execlist);
+
+                status.lite_restore = status.preempted = 1;
+                status.context_id = ctx[0].context_id;
+
+                emulate_csb_update(execlist, &status, false);
+        } else {
+                gvt_dbg_el("emulate as pending slot\n");
+                /*
+                 * otherwise
+                 * --> emulate pending execlist exist + but no preemption case
+                 */
+                execlist->pending_slot = slot;
+                emulate_execlist_status(execlist);
+        }
+        return 0;
+}
+
+static void free_workload(struct intel_vgpu_workload *workload)
+{
+        intel_vgpu_unpin_mm(workload->shadow_mm);
+        intel_gvt_mm_unreference(workload->shadow_mm);
+        kmem_cache_free(workload->vgpu->workloads, workload);
+}
+
+#define get_desc_from_elsp_dwords(ed, i) \
+        ((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
+
+
+#define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2))
+#define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U))
+static int set_gma_to_bb_cmd(struct intel_shadow_bb_entry *entry_obj,
+                             unsigned long add, int gmadr_bytes)
+{
+        if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8))
+                return -1;
+
+        *((u32 *)(entry_obj->bb_start_cmd_va + (1 << 2))) = add &
+                BATCH_BUFFER_ADDR_MASK;
+        if (gmadr_bytes == 8) {
+                *((u32 *)(entry_obj->bb_start_cmd_va + (2 << 2))) =
+                        add & BATCH_BUFFER_ADDR_HIGH_MASK;
+        }
+
+        return 0;
+}
+
+static void prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+        int gmadr_bytes = workload->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+
+        /* pin the gem object to ggtt */
+        if (!list_empty(&workload->shadow_bb)) {
+                struct intel_shadow_bb_entry *entry_obj =
+                        list_first_entry(&workload->shadow_bb,
+                                         struct intel_shadow_bb_entry,
+                                         list);
+                struct intel_shadow_bb_entry *temp;
+
+                list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb,
+                                list) {
+                        struct i915_vma *vma;
+
+                        vma = i915_gem_object_ggtt_pin(entry_obj->obj, NULL, 0,
+                                                       4, 0);
+                        if (IS_ERR(vma)) {
+                                gvt_err("Cannot pin\n");
+                                return;
+                        }
+
+                        /* FIXME: we are not tracking our pinned VMA leaving it
+                         * up to the core to fix up the stray pin_count upon
+                         * free.
+                         */
+
+                        /* update the relocate gma with shadow batch buffer*/
+                        set_gma_to_bb_cmd(entry_obj,
+                                          i915_ggtt_offset(vma),
+                                          gmadr_bytes);
+                }
+        }
+}
+
+static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        int ring_id = wa_ctx->workload->ring_id;
+        struct i915_gem_context *shadow_ctx =
+                wa_ctx->workload->vgpu->shadow_ctx;
+        struct drm_i915_gem_object *ctx_obj =
+                shadow_ctx->engine[ring_id].state->obj;
+        struct execlist_ring_context *shadow_ring_context;
+        struct page *page;
+
+        page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+        shadow_ring_context = kmap_atomic(page);
+
+        shadow_ring_context->bb_per_ctx_ptr.val =
+                (shadow_ring_context->bb_per_ctx_ptr.val &
+                (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
+        shadow_ring_context->rcs_indirect_ctx.val =
+                (shadow_ring_context->rcs_indirect_ctx.val &
+                (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
+
+        kunmap_atomic(shadow_ring_context);
+        return 0;
+}
+
+static void prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        struct i915_vma *vma;
+        unsigned char *per_ctx_va =
+                (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
+                wa_ctx->indirect_ctx.size;
+
+        if (wa_ctx->indirect_ctx.size == 0)
+                return;
+
+        vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
+                                       0, CACHELINE_BYTES, 0);
+        if (IS_ERR(vma)) {
+                gvt_err("Cannot pin indirect ctx obj\n");
+                return;
+        }
+
+        /* FIXME: we are not tracking our pinned VMA leaving it
+         * up to the core to fix up the stray pin_count upon
+         * free.
+         */
+
+        wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
+
+        wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
+        memset(per_ctx_va, 0, CACHELINE_BYTES);
+
+        update_wa_ctx_2_shadow_ctx(wa_ctx);
+}
+
+static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct execlist_ctx_descriptor_format ctx[2];
+        int ring_id = workload->ring_id;
+
+        intel_vgpu_pin_mm(workload->shadow_mm);
+        intel_vgpu_sync_oos_pages(workload->vgpu);
+        intel_vgpu_flush_post_shadow(workload->vgpu);
+        prepare_shadow_batch_buffer(workload);
+        prepare_shadow_wa_ctx(&workload->wa_ctx);
+        if (!workload->emulate_schedule_in)
+                return 0;
+
+        ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
+        ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
+
+        return emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx);
+}
+
+static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+        /* release all the shadow batch buffer */
+        if (!list_empty(&workload->shadow_bb)) {
+                struct intel_shadow_bb_entry *entry_obj =
+                        list_first_entry(&workload->shadow_bb,
+                                         struct intel_shadow_bb_entry,
+                                         list);
+                struct intel_shadow_bb_entry *temp;
+
+                list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb,
+                                         list) {
+                        i915_gem_object_unpin_map(entry_obj->obj);
+                        i915_gem_object_put(entry_obj->obj);
+                        list_del(&entry_obj->list);
+                        kfree(entry_obj);
+                }
+        }
+}
+
+static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        if (wa_ctx->indirect_ctx.size == 0)
+                return;
+
+        i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
+        i915_gem_object_put(wa_ctx->indirect_ctx.obj);
+}
+
+static int complete_execlist_workload(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_vgpu_execlist *execlist =
+                &vgpu->execlist[workload->ring_id];
+        struct intel_vgpu_workload *next_workload;
+        struct list_head *next = workload_q_head(vgpu, workload->ring_id)->next;
+        bool lite_restore = false;
+        int ret;
+
+        gvt_dbg_el("complete workload %p status %d\n", workload,
+                        workload->status);
+
+        release_shadow_batch_buffer(workload);
+        release_shadow_wa_ctx(&workload->wa_ctx);
+
+        if (workload->status || vgpu->resetting)
+                goto out;
+
+        if (!list_empty(workload_q_head(vgpu, workload->ring_id))) {
+                struct execlist_ctx_descriptor_format *this_desc, *next_desc;
+
+                next_workload = container_of(next,
+                                struct intel_vgpu_workload, list);
+                this_desc = &workload->ctx_desc;
+                next_desc = &next_workload->ctx_desc;
+
+                lite_restore = same_context(this_desc, next_desc);
+        }
+
+        if (lite_restore) {
+                gvt_dbg_el("next context == current - no schedule-out\n");
+                free_workload(workload);
+                return 0;
+        }
+
+        ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc);
+        if (ret)
+                goto err;
+out:
+        free_workload(workload);
+        return 0;
+err:
+        free_workload(workload);
+        return ret;
+}
+
+#define RING_CTX_OFF(x) \
+        offsetof(struct execlist_ring_context, x)
+
+static void read_guest_pdps(struct intel_vgpu *vgpu,
+                u64 ring_context_gpa, u32 pdp[8])
+{
+        u64 gpa;
+        int i;
+
+        gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val);
+
+        for (i = 0; i < 8; i++)
+                intel_gvt_hypervisor_read_gpa(vgpu,
+                                gpa + i * 8, &pdp[7 - i], 4);
+}
+
+static int prepare_mm(struct intel_vgpu_workload *workload)
+{
+        struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
+        struct intel_vgpu_mm *mm;
+        int page_table_level;
+        u32 pdp[8];
+
+        if (desc->addressing_mode == 1) { /* legacy 32-bit */
+                page_table_level = 3;
+        } else if (desc->addressing_mode == 3) { /* legacy 64 bit */
+                page_table_level = 4;
+        } else {
+                gvt_err("Advanced Context mode(SVM) is not supported!\n");
+                return -EINVAL;
+        }
+
+        read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp);
+
+        mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp);
+        if (mm) {
+                intel_gvt_mm_reference(mm);
+        } else {
+
+                mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT,
+                                pdp, page_table_level, 0);
+                if (IS_ERR(mm)) {
+                        gvt_err("fail to create mm object.\n");
+                        return PTR_ERR(mm);
+                }
+        }
+        workload->shadow_mm = mm;
+        return 0;
+}
+
+#define get_last_workload(q) \
+        (list_empty(q) ? NULL : container_of(q->prev, \
+        struct intel_vgpu_workload, list))
+
+static int submit_context(struct intel_vgpu *vgpu, int ring_id,
+                struct execlist_ctx_descriptor_format *desc,
+                bool emulate_schedule_in)
+{
+        struct list_head *q = workload_q_head(vgpu, ring_id);
+        struct intel_vgpu_workload *last_workload = get_last_workload(q);
+        struct intel_vgpu_workload *workload = NULL;
+        u64 ring_context_gpa;
+        u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
+        int ret;
+
+        ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+                        (u32)((desc->lrca + 1) << GTT_PAGE_SHIFT));
+        if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
+                gvt_err("invalid guest context LRCA: %x\n", desc->lrca);
+                return -EINVAL;
+        }
+
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(ring_header.val), &head, 4);
+
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(ring_tail.val), &tail, 4);
+
+        head &= RB_HEAD_OFF_MASK;
+        tail &= RB_TAIL_OFF_MASK;
+
+        if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
+                gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
+                gvt_dbg_el("ctx head %x real head %lx\n", head,
+                                last_workload->rb_tail);
+                /*
+                 * cannot use guest context head pointer here,
+                 * as it might not be updated at this time
+                 */
+                head = last_workload->rb_tail;
+        }
+
+        gvt_dbg_el("ring id %d begin a new workload\n", ring_id);
+
+        workload = kmem_cache_zalloc(vgpu->workloads, GFP_KERNEL);
+        if (!workload)
+                return -ENOMEM;
+
+        /* record some ring buffer register values for scan and shadow */
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(rb_start.val), &start, 4);
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
+
+        INIT_LIST_HEAD(&workload->list);
+        INIT_LIST_HEAD(&workload->shadow_bb);
+
+        init_waitqueue_head(&workload->shadow_ctx_status_wq);
+        atomic_set(&workload->shadow_ctx_active, 0);
+
+        workload->vgpu = vgpu;
+        workload->ring_id = ring_id;
+        workload->ctx_desc = *desc;
+        workload->ring_context_gpa = ring_context_gpa;
+        workload->rb_head = head;
+        workload->rb_tail = tail;
+        workload->rb_start = start;
+        workload->rb_ctl = ctl;
+        workload->prepare = prepare_execlist_workload;
+        workload->complete = complete_execlist_workload;
+        workload->status = -EINPROGRESS;
+        workload->emulate_schedule_in = emulate_schedule_in;
+
+        if (ring_id == RCS) {
+                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
+                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
+
+                workload->wa_ctx.indirect_ctx.guest_gma =
+                        indirect_ctx & INDIRECT_CTX_ADDR_MASK;
+                workload->wa_ctx.indirect_ctx.size =
+                        (indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
+                        CACHELINE_BYTES;
+                workload->wa_ctx.per_ctx.guest_gma =
+                        per_ctx & PER_CTX_ADDR_MASK;
+                workload->wa_ctx.workload = workload;
+
+                WARN_ON(workload->wa_ctx.indirect_ctx.size && !(per_ctx & 0x1));
+        }
+
+        if (emulate_schedule_in)
+                memcpy(&workload->elsp_dwords,
+                                &vgpu->execlist[ring_id].elsp_dwords,
+                                sizeof(workload->elsp_dwords));
+
+        gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
+                        workload, ring_id, head, tail, start, ctl);
+
+        gvt_dbg_el("workload %p emulate schedule_in %d\n", workload,
+                        emulate_schedule_in);
+
+        ret = prepare_mm(workload);
+        if (ret) {
+                kmem_cache_free(vgpu->workloads, workload);
+                return ret;
+        }
+
+        queue_workload(workload);
+        return 0;
+}
+
+int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+        struct execlist_ctx_descriptor_format *desc[2], valid_desc[2];
+        unsigned long valid_desc_bitmap = 0;
+        bool emulate_schedule_in = true;
+        int ret;
+        int i;
+
+        memset(valid_desc, 0, sizeof(valid_desc));
+
+        desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
+        desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
+
+        for (i = 0; i < 2; i++) {
+                if (!desc[i]->valid)
+                        continue;
+
+                if (!desc[i]->privilege_access) {
+                        gvt_err("vgpu%d: unexpected GGTT elsp submission\n",
+                                        vgpu->id);
+                        return -EINVAL;
+                }
+
+                /* TODO: add another guest context checks here. */
+                set_bit(i, &valid_desc_bitmap);
+                valid_desc[i] = *desc[i];
+        }
+
+        if (!valid_desc_bitmap) {
+                gvt_err("vgpu%d: no valid desc in a elsp submission\n",
+                                vgpu->id);
+                return -EINVAL;
+        }
+
+        if (!test_bit(0, (void *)&valid_desc_bitmap) &&
+                        test_bit(1, (void *)&valid_desc_bitmap)) {
+                gvt_err("vgpu%d: weird elsp submission, desc 0 is not valid\n",
+                                vgpu->id);
+                return -EINVAL;
+        }
+
+        /* submit workload */
+        for_each_set_bit(i, (void *)&valid_desc_bitmap, 2) {
+                ret = submit_context(vgpu, ring_id, &valid_desc[i],
+                                emulate_schedule_in);
+                if (ret) {
+                        gvt_err("vgpu%d: fail to schedule workload\n",
+                                        vgpu->id);
+                        return ret;
+                }
+                emulate_schedule_in = false;
+        }
+        return 0;
+}
+
+static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+        struct execlist_context_status_pointer_format ctx_status_ptr;
+        u32 ctx_status_ptr_reg;
+
+        memset(execlist, 0, sizeof(*execlist));
+
+        execlist->vgpu = vgpu;
+        execlist->ring_id = ring_id;
+        execlist->slot[0].index = 0;
+        execlist->slot[1].index = 1;
+
+        ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+                        _EL_OFFSET_STATUS_PTR);
+
+        ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
+        ctx_status_ptr.read_ptr = ctx_status_ptr.write_ptr = 0x7;
+        vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
+}
+
+void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu)
+{
+        kmem_cache_destroy(vgpu->workloads);
+}
+
+int intel_vgpu_init_execlist(struct intel_vgpu *vgpu)
+{
+        enum intel_engine_id i;
+        struct intel_engine_cs *engine;
+
+        /* each ring has a virtual execlist engine */
+        for_each_engine(engine, vgpu->gvt->dev_priv, i) {
+                init_vgpu_execlist(vgpu, i);
+                INIT_LIST_HEAD(&vgpu->workload_q_head[i]);
+        }
+
+        vgpu->workloads = kmem_cache_create("gvt-g vgpu workload",
+                        sizeof(struct intel_vgpu_workload), 0,
+                        SLAB_HWCACHE_ALIGN,
+                        NULL);
+
+        if (!vgpu->workloads)
+                return -ENOMEM;
+
+        return 0;
+}
+
+void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
+                unsigned long ring_bitmap)
+{
+        int bit;
+        struct list_head *pos, *n;
+        struct intel_vgpu_workload *workload = NULL;
+
+        for_each_set_bit(bit, &ring_bitmap, sizeof(ring_bitmap) * 8) {
+                if (bit >= I915_NUM_ENGINES)
+                        break;
+                /* free the unsubmited workload in the queue */
+                list_for_each_safe(pos, n, &vgpu->workload_q_head[bit]) {
+                        workload = container_of(pos,
+                                        struct intel_vgpu_workload, list);
+                        list_del_init(&workload->list);
+                        free_workload(workload);
+                }
+
+                init_vgpu_execlist(vgpu, bit);
+        }
+}
diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h
new file mode 100644
index 000000000000..635f31c6dcc1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/execlist.h
@@ -0,0 +1,188 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *
+ */
+
+#ifndef _GVT_EXECLIST_H_
+#define _GVT_EXECLIST_H_
+
+struct execlist_ctx_descriptor_format {
+        union {
+                u32 udw;
+                u32 context_id;
+        };
+        union {
+                u32 ldw;
+                struct {
+                        u32 valid                  : 1;
+                        u32 force_pd_restore       : 1;
+                        u32 force_restore          : 1;
+                        u32 addressing_mode        : 2;
+                        u32 llc_coherency          : 1;
+                        u32 fault_handling         : 2;
+                        u32 privilege_access       : 1;
+                        u32 reserved               : 3;
+                        u32 lrca                   : 20;
+                };
+        };
+};
+
+struct execlist_status_format {
+        union {
+                u32 ldw;
+                struct {
+                        u32 current_execlist_pointer       :1;
+                        u32 execlist_write_pointer         :1;
+                        u32 execlist_queue_full            :1;
+                        u32 execlist_1_valid               :1;
+                        u32 execlist_0_valid               :1;
+                        u32 last_ctx_switch_reason         :9;
+                        u32 current_active_elm_status      :2;
+                        u32 arbitration_enable             :1;
+                        u32 execlist_1_active              :1;
+                        u32 execlist_0_active              :1;
+                        u32 reserved                       :13;
+                };
+        };
+        union {
+                u32 udw;
+                u32 context_id;
+        };
+};
+
+struct execlist_context_status_pointer_format {
+        union {
+                u32 dw;
+                struct {
+                        u32 write_ptr              :3;
+                        u32 reserved               :5;
+                        u32 read_ptr               :3;
+                        u32 reserved2              :5;
+                        u32 mask                   :16;
+                };
+        };
+};
+
+struct execlist_context_status_format {
+        union {
+                u32 ldw;
+                struct {
+                        u32 idle_to_active         :1;
+                        u32 preempted              :1;
+                        u32 element_switch         :1;
+                        u32 active_to_idle         :1;
+                        u32 context_complete       :1;
+                        u32 wait_on_sync_flip      :1;
+                        u32 wait_on_vblank         :1;
+                        u32 wait_on_semaphore      :1;
+                        u32 wait_on_scanline       :1;
+                        u32 reserved               :2;
+                        u32 semaphore_wait_mode    :1;
+                        u32 display_plane          :3;
+                        u32 lite_restore           :1;
+                        u32 reserved_2             :16;
+                };
+        };
+        union {
+                u32 udw;
+                u32 context_id;
+        };
+};
+
+struct execlist_mmio_pair {
+        u32 addr;
+        u32 val;
+};
+
+/* The first 52 dwords in register state context */
+struct execlist_ring_context {
+        u32 nop1;
+        u32 lri_cmd_1;
+        struct execlist_mmio_pair ctx_ctrl;
+        struct execlist_mmio_pair ring_header;
+        struct execlist_mmio_pair ring_tail;
+        struct execlist_mmio_pair rb_start;
+        struct execlist_mmio_pair rb_ctrl;
+        struct execlist_mmio_pair bb_cur_head_UDW;
+        struct execlist_mmio_pair bb_cur_head_LDW;
+        struct execlist_mmio_pair bb_state;
+        struct execlist_mmio_pair second_bb_addr_UDW;
+        struct execlist_mmio_pair second_bb_addr_LDW;
+        struct execlist_mmio_pair second_bb_state;
+        struct execlist_mmio_pair bb_per_ctx_ptr;
+        struct execlist_mmio_pair rcs_indirect_ctx;
+        struct execlist_mmio_pair rcs_indirect_ctx_offset;
+        u32 nop2;
+        u32 nop3;
+        u32 nop4;
+        u32 lri_cmd_2;
+        struct execlist_mmio_pair ctx_timestamp;
+        struct execlist_mmio_pair pdp3_UDW;
+        struct execlist_mmio_pair pdp3_LDW;
+        struct execlist_mmio_pair pdp2_UDW;
+        struct execlist_mmio_pair pdp2_LDW;
+        struct execlist_mmio_pair pdp1_UDW;
+        struct execlist_mmio_pair pdp1_LDW;
+        struct execlist_mmio_pair pdp0_UDW;
+        struct execlist_mmio_pair pdp0_LDW;
+};
+
+struct intel_vgpu_elsp_dwords {
+        u32 data[4];
+        u32 index;
+};
+
+struct intel_vgpu_execlist_slot {
+        struct execlist_ctx_descriptor_format ctx[2];
+        u32 index;
+};
+
+struct intel_vgpu_execlist {
+        struct intel_vgpu_execlist_slot slot[2];
+        struct intel_vgpu_execlist_slot *running_slot;
+        struct intel_vgpu_execlist_slot *pending_slot;
+        struct execlist_ctx_descriptor_format *running_context;
+        int ring_id;
+        struct intel_vgpu *vgpu;
+        struct intel_vgpu_elsp_dwords elsp_dwords;
+};
+
+void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu);
+
+int intel_vgpu_init_execlist(struct intel_vgpu *vgpu);
+
+int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id);
+
+void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
+                unsigned long ring_bitmap);
+
+#endif /*_GVT_EXECLIST_H_*/
diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c
new file mode 100644
index 000000000000..2fae2a2ca96f
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/firmware.c
@@ -0,0 +1,312 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Changbin Du <changbin.du@intel.com>
+ *
+ */
+
+#include <linux/firmware.h>
+#include <linux/crc32.h>
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+
+#define FIRMWARE_VERSION (0x0)
+
+struct gvt_firmware_header {
+        u64 magic;
+        u32 crc32;              /* protect the data after this field */
+        u32 version;
+        u64 cfg_space_size;
+        u64 cfg_space_offset;   /* offset in the file */
+        u64 mmio_size;
+        u64 mmio_offset;        /* offset in the file */
+        unsigned char data[1];
+};
+
+#define RD(offset) (readl(mmio + offset.reg))
+#define WR(v, offset) (writel(v, mmio + offset.reg))
+
+static void bdw_forcewake_get(void __iomem *mmio)
+{
+        WR(_MASKED_BIT_DISABLE(0xffff), FORCEWAKE_MT);
+
+        RD(ECOBUS);
+
+        if (wait_for((RD(FORCEWAKE_ACK_HSW) & FORCEWAKE_KERNEL) == 0, 50))
+                gvt_err("fail to wait forcewake idle\n");
+
+        WR(_MASKED_BIT_ENABLE(FORCEWAKE_KERNEL), FORCEWAKE_MT);
+
+        if (wait_for((RD(FORCEWAKE_ACK_HSW) & FORCEWAKE_KERNEL), 50))
+                gvt_err("fail to wait forcewake ack\n");
+
+        if (wait_for((RD(GEN6_GT_THREAD_STATUS_REG) &
+                      GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 50))
+                gvt_err("fail to wait c0 wake up\n");
+}
+
+#undef RD
+#undef WR
+
+#define dev_to_drm_minor(d) dev_get_drvdata((d))
+
+static ssize_t
+gvt_firmware_read(struct file *filp, struct kobject *kobj,
+             struct bin_attribute *attr, char *buf,
+             loff_t offset, size_t count)
+{
+        memcpy(buf, attr->private + offset, count);
+        return count;
+}
+
+static struct bin_attribute firmware_attr = {
+        .attr = {.name = "gvt_firmware", .mode = (S_IRUSR)},
+        .read = gvt_firmware_read,
+        .write = NULL,
+        .mmap = NULL,
+};
+
+static int expose_firmware_sysfs(struct intel_gvt *gvt,
+                                        void __iomem *mmio)
+{
+        struct intel_gvt_device_info *info = &gvt->device_info;
+        struct pci_dev *pdev = gvt->dev_priv->drm.pdev;
+        struct intel_gvt_mmio_info *e;
+        struct gvt_firmware_header *h;
+        void *firmware;
+        void *p;
+        unsigned long size;
+        int i;
+        int ret;
+
+        size = sizeof(*h) + info->mmio_size + info->cfg_space_size - 1;
+        firmware = vmalloc(size);
+        if (!firmware)
+                return -ENOMEM;
+
+        h = firmware;
+
+        h->magic = VGT_MAGIC;
+        h->version = FIRMWARE_VERSION;
+        h->cfg_space_size = info->cfg_space_size;
+        h->cfg_space_offset = offsetof(struct gvt_firmware_header, data);
+        h->mmio_size = info->mmio_size;
+        h->mmio_offset = h->cfg_space_offset + h->cfg_space_size;
+
+        p = firmware + h->cfg_space_offset;
+
+        for (i = 0; i < h->cfg_space_size; i += 4)
+                pci_read_config_dword(pdev, i, p + i);
+
+        memcpy(gvt->firmware.cfg_space, p, info->cfg_space_size);
+
+        p = firmware + h->mmio_offset;
+
+        hash_for_each(gvt->mmio.mmio_info_table, i, e, node) {
+                int j;
+
+                for (j = 0; j < e->length; j += 4)
+                        *(u32 *)(p + e->offset + j) =
+                                readl(mmio + e->offset + j);
+        }
+
+        memcpy(gvt->firmware.mmio, p, info->mmio_size);
+
+        firmware_attr.size = size;
+        firmware_attr.private = firmware;
+
+        ret = device_create_bin_file(&pdev->dev, &firmware_attr);
+        if (ret) {
+                vfree(firmware);
+                return ret;
+        }
+        return 0;
+}
+
+static void clean_firmware_sysfs(struct intel_gvt *gvt)
+{
+        struct pci_dev *pdev = gvt->dev_priv->drm.pdev;
+
+        device_remove_bin_file(&pdev->dev, &firmware_attr);
+        vfree(firmware_attr.private);
+}
+
+/**
+ * intel_gvt_free_firmware - free GVT firmware
+ * @gvt: intel gvt device
+ *
+ */
+void intel_gvt_free_firmware(struct intel_gvt *gvt)
+{
+        if (!gvt->firmware.firmware_loaded)
+                clean_firmware_sysfs(gvt);
+
+        kfree(gvt->firmware.cfg_space);
+        kfree(gvt->firmware.mmio);
+}
+
+static int verify_firmware(struct intel_gvt *gvt,
+                           const struct firmware *fw)
+{
+        struct intel_gvt_device_info *info = &gvt->device_info;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
+        struct gvt_firmware_header *h;
+        unsigned long id, crc32_start;
+        const void *mem;
+        const char *item;
+        u64 file, request;
+
+        h = (struct gvt_firmware_header *)fw->data;
+
+        crc32_start = offsetof(struct gvt_firmware_header, crc32) + 4;
+        mem = fw->data + crc32_start;
+
+#define VERIFY(s, a, b) do { \
+        item = (s); file = (u64)(a); request = (u64)(b); \
+        if ((a) != (b)) \
+                goto invalid_firmware; \
+} while (0)
+
+        VERIFY("magic number", h->magic, VGT_MAGIC);
+        VERIFY("version", h->version, FIRMWARE_VERSION);
+        VERIFY("crc32", h->crc32, crc32_le(0, mem, fw->size - crc32_start));
+        VERIFY("cfg space size", h->cfg_space_size, info->cfg_space_size);
+        VERIFY("mmio size", h->mmio_size, info->mmio_size);
+
+        mem = (fw->data + h->cfg_space_offset);
+
+        id = *(u16 *)(mem + PCI_VENDOR_ID);
+        VERIFY("vender id", id, pdev->vendor);
+
+        id = *(u16 *)(mem + PCI_DEVICE_ID);
+        VERIFY("device id", id, pdev->device);
+
+        id = *(u8 *)(mem + PCI_REVISION_ID);
+        VERIFY("revision id", id, pdev->revision);
+
+#undef VERIFY
+        return 0;
+
+invalid_firmware:
+        gvt_dbg_core("Invalid firmware: %s [file] 0x%llx [request] 0x%llx\n",
+                     item, file, request);
+        return -EINVAL;
+}
+
+#define GVT_FIRMWARE_PATH "i915/gvt"
+
+/**
+ * intel_gvt_load_firmware - load GVT firmware
+ * @gvt: intel gvt device
+ *
+ */
+int intel_gvt_load_firmware(struct intel_gvt *gvt)
+{
+        struct intel_gvt_device_info *info = &gvt->device_info;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
+        struct intel_gvt_firmware *firmware = &gvt->firmware;
+        struct gvt_firmware_header *h;
+        const struct firmware *fw;
+        char *path;
+        void __iomem *mmio;
+        void *mem;
+        int ret;
+
+        path = kmalloc(PATH_MAX, GFP_KERNEL);
+        if (!path)
+                return -ENOMEM;
+
+        mem = kmalloc(info->cfg_space_size, GFP_KERNEL);
+        if (!mem) {
+                kfree(path);
+                return -ENOMEM;
+        }
+
+        firmware->cfg_space = mem;
+
+        mem = kmalloc(info->mmio_size, GFP_KERNEL);
+        if (!mem) {
+                kfree(path);
+                kfree(firmware->cfg_space);
+                return -ENOMEM;
+        }
+
+        firmware->mmio = mem;
+
+        mmio = pci_iomap(pdev, info->mmio_bar, info->mmio_size);
+        if (!mmio) {
+                kfree(path);
+                kfree(firmware->cfg_space);
+                kfree(firmware->mmio);
+                return -EINVAL;
+        }
+
+        if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv))
+                bdw_forcewake_get(mmio);
+
+        sprintf(path, "%s/vid_0x%04x_did_0x%04x_rid_0x%04x.golden_hw_state",
+                 GVT_FIRMWARE_PATH, pdev->vendor, pdev->device,
+                 pdev->revision);
+
+        gvt_dbg_core("request hw state firmware %s...\n", path);
+
+        ret = request_firmware(&fw, path, &dev_priv->drm.pdev->dev);
+        kfree(path);
+
+        if (ret)
+                goto expose_firmware;
+
+        gvt_dbg_core("success.\n");
+
+        ret = verify_firmware(gvt, fw);
+        if (ret)
+                goto out_free_fw;
+
+        gvt_dbg_core("verified.\n");
+
+        h = (struct gvt_firmware_header *)fw->data;
+
+        memcpy(firmware->cfg_space, fw->data + h->cfg_space_offset,
+               h->cfg_space_size);
+        memcpy(firmware->mmio, fw->data + h->mmio_offset,
+               h->mmio_size);
+
+        release_firmware(fw);
+        firmware->firmware_loaded = true;
+        pci_iounmap(pdev, mmio);
+        return 0;
+
+out_free_fw:
+        release_firmware(fw);
+expose_firmware:
+        expose_firmware_sysfs(gvt, mmio);
+        pci_iounmap(pdev, mmio);
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
new file mode 100644
index 000000000000..2cc761328569
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -0,0 +1,2232 @@
+/*
+ * GTT virtualization
+ *
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Xiao Zheng <xiao.zheng@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+#include "trace.h"
+
+static bool enable_out_of_sync = false;
+static int preallocated_oos_pages = 8192;
+
+/*
+ * validate a gm address and related range size,
+ * translate it to host gm address
+ */
+bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size)
+{
+        if ((!vgpu_gmadr_is_valid(vgpu, addr)) || (size
+                        && !vgpu_gmadr_is_valid(vgpu, addr + size - 1))) {
+                gvt_err("vgpu%d: invalid range gmadr 0x%llx size 0x%x\n",
+                                vgpu->id, addr, size);
+                return false;
+        }
+        return true;
+}
+
+/* translate a guest gmadr to host gmadr */
+int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr)
+{
+        if (WARN(!vgpu_gmadr_is_valid(vgpu, g_addr),
+                 "invalid guest gmadr %llx\n", g_addr))
+                return -EACCES;
+
+        if (vgpu_gmadr_is_aperture(vgpu, g_addr))
+                *h_addr = vgpu_aperture_gmadr_base(vgpu)
+                          + (g_addr - vgpu_aperture_offset(vgpu));
+        else
+                *h_addr = vgpu_hidden_gmadr_base(vgpu)
+                          + (g_addr - vgpu_hidden_offset(vgpu));
+        return 0;
+}
+
+/* translate a host gmadr to guest gmadr */
+int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr)
+{
+        if (WARN(!gvt_gmadr_is_valid(vgpu->gvt, h_addr),
+                 "invalid host gmadr %llx\n", h_addr))
+                return -EACCES;
+
+        if (gvt_gmadr_is_aperture(vgpu->gvt, h_addr))
+                *g_addr = vgpu_aperture_gmadr_base(vgpu)
+                        + (h_addr - gvt_aperture_gmadr_base(vgpu->gvt));
+        else
+                *g_addr = vgpu_hidden_gmadr_base(vgpu)
+                        + (h_addr - gvt_hidden_gmadr_base(vgpu->gvt));
+        return 0;
+}
+
+int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
+                             unsigned long *h_index)
+{
+        u64 h_addr;
+        int ret;
+
+        ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << GTT_PAGE_SHIFT,
+                                       &h_addr);
+        if (ret)
+                return ret;
+
+        *h_index = h_addr >> GTT_PAGE_SHIFT;
+        return 0;
+}
+
+int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
+                             unsigned long *g_index)
+{
+        u64 g_addr;
+        int ret;
+
+        ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << GTT_PAGE_SHIFT,
+                                       &g_addr);
+        if (ret)
+                return ret;
+
+        *g_index = g_addr >> GTT_PAGE_SHIFT;
+        return 0;
+}
+
+#define gtt_type_is_entry(type) \
+        (type > GTT_TYPE_INVALID && type < GTT_TYPE_PPGTT_ENTRY \
+         && type != GTT_TYPE_PPGTT_PTE_ENTRY \
+         && type != GTT_TYPE_PPGTT_ROOT_ENTRY)
+
+#define gtt_type_is_pt(type) \
+        (type >= GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX)
+
+#define gtt_type_is_pte_pt(type) \
+        (type == GTT_TYPE_PPGTT_PTE_PT)
+
+#define gtt_type_is_root_pointer(type) \
+        (gtt_type_is_entry(type) && type > GTT_TYPE_PPGTT_ROOT_ENTRY)
+
+#define gtt_init_entry(e, t, p, v) do { \
+        (e)->type = t; \
+        (e)->pdev = p; \
+        memcpy(&(e)->val64, &v, sizeof(v)); \
+} while (0)
+
+enum {
+        GTT_TYPE_INVALID = -1,
+
+        GTT_TYPE_GGTT_PTE,
+
+        GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+        GTT_TYPE_PPGTT_PTE_2M_ENTRY,
+        GTT_TYPE_PPGTT_PTE_1G_ENTRY,
+
+        GTT_TYPE_PPGTT_PTE_ENTRY,
+
+        GTT_TYPE_PPGTT_PDE_ENTRY,
+        GTT_TYPE_PPGTT_PDP_ENTRY,
+        GTT_TYPE_PPGTT_PML4_ENTRY,
+
+        GTT_TYPE_PPGTT_ROOT_ENTRY,
+
+        GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+        GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+
+        GTT_TYPE_PPGTT_ENTRY,
+
+        GTT_TYPE_PPGTT_PTE_PT,
+        GTT_TYPE_PPGTT_PDE_PT,
+        GTT_TYPE_PPGTT_PDP_PT,
+        GTT_TYPE_PPGTT_PML4_PT,
+
+        GTT_TYPE_MAX,
+};
+
+/*
+ * Mappings between GTT_TYPE* enumerations.
+ * Following information can be found according to the given type:
+ * - type of next level page table
+ * - type of entry inside this level page table
+ * - type of entry with PSE set
+ *
+ * If the given type doesn't have such a kind of information,
+ * e.g. give a l4 root entry type, then request to get its PSE type,
+ * give a PTE page table type, then request to get its next level page
+ * table type, as we know l4 root entry doesn't have a PSE bit,
+ * and a PTE page table doesn't have a next level page table type,
+ * GTT_TYPE_INVALID will be returned. This is useful when traversing a
+ * page table.
+ */
+
+struct gtt_type_table_entry {
+        int entry_type;
+        int next_pt_type;
+        int pse_entry_type;
+};
+
+#define GTT_TYPE_TABLE_ENTRY(type, e_type, npt_type, pse_type) \
+        [type] = { \
+                .entry_type = e_type, \
+                .next_pt_type = npt_type, \
+                .pse_entry_type = pse_type, \
+        }
+
+static struct gtt_type_table_entry gtt_type_table[] = {
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+                        GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+                        GTT_TYPE_PPGTT_PML4_PT,
+                        GTT_TYPE_INVALID),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_PT,
+                        GTT_TYPE_PPGTT_PML4_ENTRY,
+                        GTT_TYPE_PPGTT_PDP_PT,
+                        GTT_TYPE_INVALID),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_ENTRY,
+                        GTT_TYPE_PPGTT_PML4_ENTRY,
+                        GTT_TYPE_PPGTT_PDP_PT,
+                        GTT_TYPE_INVALID),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_PT,
+                        GTT_TYPE_PPGTT_PDP_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_PT,
+                        GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+                        GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_PT,
+                        GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_ENTRY,
+                        GTT_TYPE_PPGTT_PDP_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_PT,
+                        GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_PT,
+                        GTT_TYPE_PPGTT_PDE_ENTRY,
+                        GTT_TYPE_PPGTT_PTE_PT,
+                        GTT_TYPE_PPGTT_PTE_2M_ENTRY),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_ENTRY,
+                        GTT_TYPE_PPGTT_PTE_PT,
+                        GTT_TYPE_PPGTT_PTE_2M_ENTRY),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT,
+                        GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+                        GTT_TYPE_INVALID,
+                        GTT_TYPE_INVALID),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+                        GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+                        GTT_TYPE_INVALID,
+                        GTT_TYPE_INVALID),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_ENTRY,
+                        GTT_TYPE_INVALID,
+                        GTT_TYPE_PPGTT_PTE_2M_ENTRY),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_1G_ENTRY,
+                        GTT_TYPE_PPGTT_PDP_ENTRY,
+                        GTT_TYPE_INVALID,
+                        GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+        GTT_TYPE_TABLE_ENTRY(GTT_TYPE_GGTT_PTE,
+                        GTT_TYPE_GGTT_PTE,
+                        GTT_TYPE_INVALID,
+                        GTT_TYPE_INVALID),
+};
+
+static inline int get_next_pt_type(int type)
+{
+        return gtt_type_table[type].next_pt_type;
+}
+
+static inline int get_entry_type(int type)
+{
+        return gtt_type_table[type].entry_type;
+}
+
+static inline int get_pse_type(int type)
+{
+        return gtt_type_table[type].pse_entry_type;
+}
+
+static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index)
+{
+        void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
+        u64 pte;
+
+#ifdef readq
+        pte = readq(addr);
+#else
+        pte = ioread32(addr);
+        pte |= ioread32(addr + 4) << 32;
+#endif
+        return pte;
+}
+
+static void write_pte64(struct drm_i915_private *dev_priv,
+                unsigned long index, u64 pte)
+{
+        void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
+
+#ifdef writeq
+        writeq(pte, addr);
+#else
+        iowrite32((u32)pte, addr);
+        iowrite32(pte >> 32, addr + 4);
+#endif
+        I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+        POSTING_READ(GFX_FLSH_CNTL_GEN6);
+}
+
+static inline struct intel_gvt_gtt_entry *gtt_get_entry64(void *pt,
+                struct intel_gvt_gtt_entry *e,
+                unsigned long index, bool hypervisor_access, unsigned long gpa,
+                struct intel_vgpu *vgpu)
+{
+        const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+        int ret;
+
+        if (WARN_ON(info->gtt_entry_size != 8))
+                return e;
+
+        if (hypervisor_access) {
+                ret = intel_gvt_hypervisor_read_gpa(vgpu, gpa +
+                                (index << info->gtt_entry_size_shift),
+                                &e->val64, 8);
+                WARN_ON(ret);
+        } else if (!pt) {
+                e->val64 = read_pte64(vgpu->gvt->dev_priv, index);
+        } else {
+                e->val64 = *((u64 *)pt + index);
+        }
+        return e;
+}
+
+static inline struct intel_gvt_gtt_entry *gtt_set_entry64(void *pt,
+                struct intel_gvt_gtt_entry *e,
+                unsigned long index, bool hypervisor_access, unsigned long gpa,
+                struct intel_vgpu *vgpu)
+{
+        const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+        int ret;
+
+        if (WARN_ON(info->gtt_entry_size != 8))
+                return e;
+
+        if (hypervisor_access) {
+                ret = intel_gvt_hypervisor_write_gpa(vgpu, gpa +
+                                (index << info->gtt_entry_size_shift),
+                                &e->val64, 8);
+                WARN_ON(ret);
+        } else if (!pt) {
+                write_pte64(vgpu->gvt->dev_priv, index, e->val64);
+        } else {
+                *((u64 *)pt + index) = e->val64;
+        }
+        return e;
+}
+
+#define GTT_HAW 46
+
+#define ADDR_1G_MASK (((1UL << (GTT_HAW - 30 + 1)) - 1) << 30)
+#define ADDR_2M_MASK (((1UL << (GTT_HAW - 21 + 1)) - 1) << 21)
+#define ADDR_4K_MASK (((1UL << (GTT_HAW - 12 + 1)) - 1) << 12)
+
+static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e)
+{
+        unsigned long pfn;
+
+        if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY)
+                pfn = (e->val64 & ADDR_1G_MASK) >> 12;
+        else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY)
+                pfn = (e->val64 & ADDR_2M_MASK) >> 12;
+        else
+                pfn = (e->val64 & ADDR_4K_MASK) >> 12;
+        return pfn;
+}
+
+static void gen8_gtt_set_pfn(struct intel_gvt_gtt_entry *e, unsigned long pfn)
+{
+        if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) {
+                e->val64 &= ~ADDR_1G_MASK;
+                pfn &= (ADDR_1G_MASK >> 12);
+        } else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) {
+                e->val64 &= ~ADDR_2M_MASK;
+                pfn &= (ADDR_2M_MASK >> 12);
+        } else {
+                e->val64 &= ~ADDR_4K_MASK;
+                pfn &= (ADDR_4K_MASK >> 12);
+        }
+
+        e->val64 |= (pfn << 12);
+}
+
+static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
+{
+        /* Entry doesn't have PSE bit. */
+        if (get_pse_type(e->type) == GTT_TYPE_INVALID)
+                return false;
+
+        e->type = get_entry_type(e->type);
+        if (!(e->val64 & (1 << 7)))
+                return false;
+
+        e->type = get_pse_type(e->type);
+        return true;
+}
+
+static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
+{
+        /*
+         * i915 writes PDP root pointer registers without present bit,
+         * it also works, so we need to treat root pointer entry
+         * specifically.
+         */
+        if (e->type == GTT_TYPE_PPGTT_ROOT_L3_ENTRY
+                        || e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
+                return (e->val64 != 0);
+        else
+                return (e->val64 & (1 << 0));
+}
+
+static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
+{
+        e->val64 &= ~(1 << 0);
+}
+
+/*
+ * Per-platform GMA routines.
+ */
+static unsigned long gma_to_ggtt_pte_index(unsigned long gma)
+{
+        unsigned long x = (gma >> GTT_PAGE_SHIFT);
+
+        trace_gma_index(__func__, gma, x);
+        return x;
+}
+
+#define DEFINE_PPGTT_GMA_TO_INDEX(prefix, ename, exp) \
+static unsigned long prefix##_gma_to_##ename##_index(unsigned long gma) \
+{ \
+        unsigned long x = (exp); \
+        trace_gma_index(__func__, gma, x); \
+        return x; \
+}
+
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, pte, (gma >> 12 & 0x1ff));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, pde, (gma >> 21 & 0x1ff));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, l3_pdp, (gma >> 30 & 0x3));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, l4_pdp, (gma >> 30 & 0x1ff));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, pml4, (gma >> 39 & 0x1ff));
+
+static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
+        .get_entry = gtt_get_entry64,
+        .set_entry = gtt_set_entry64,
+        .clear_present = gtt_entry_clear_present,
+        .test_present = gen8_gtt_test_present,
+        .test_pse = gen8_gtt_test_pse,
+        .get_pfn = gen8_gtt_get_pfn,
+        .set_pfn = gen8_gtt_set_pfn,
+};
+
+static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
+        .gma_to_ggtt_pte_index = gma_to_ggtt_pte_index,
+        .gma_to_pte_index = gen8_gma_to_pte_index,
+        .gma_to_pde_index = gen8_gma_to_pde_index,
+        .gma_to_l3_pdp_index = gen8_gma_to_l3_pdp_index,
+        .gma_to_l4_pdp_index = gen8_gma_to_l4_pdp_index,
+        .gma_to_pml4_index = gen8_gma_to_pml4_index,
+};
+
+static int gtt_entry_p2m(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *p,
+                struct intel_gvt_gtt_entry *m)
+{
+        struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+        unsigned long gfn, mfn;
+
+        *m = *p;
+
+        if (!ops->test_present(p))
+                return 0;
+
+        gfn = ops->get_pfn(p);
+
+        mfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, gfn);
+        if (mfn == INTEL_GVT_INVALID_ADDR) {
+                gvt_err("fail to translate gfn: 0x%lx\n", gfn);
+                return -ENXIO;
+        }
+
+        ops->set_pfn(m, mfn);
+        return 0;
+}
+
+/*
+ * MM helpers.
+ */
+struct intel_gvt_gtt_entry *intel_vgpu_mm_get_entry(struct intel_vgpu_mm *mm,
+                void *page_table, struct intel_gvt_gtt_entry *e,
+                unsigned long index)
+{
+        struct intel_gvt *gvt = mm->vgpu->gvt;
+        struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+        e->type = mm->page_table_entry_type;
+
+        ops->get_entry(page_table, e, index, false, 0, mm->vgpu);
+        ops->test_pse(e);
+        return e;
+}
+
+struct intel_gvt_gtt_entry *intel_vgpu_mm_set_entry(struct intel_vgpu_mm *mm,
+                void *page_table, struct intel_gvt_gtt_entry *e,
+                unsigned long index)
+{
+        struct intel_gvt *gvt = mm->vgpu->gvt;
+        struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+        return ops->set_entry(page_table, e, index, false, 0, mm->vgpu);
+}
+
+/*
+ * PPGTT shadow page table helpers.
+ */
+static inline struct intel_gvt_gtt_entry *ppgtt_spt_get_entry(
+                struct intel_vgpu_ppgtt_spt *spt,
+                void *page_table, int type,
+                struct intel_gvt_gtt_entry *e, unsigned long index,
+                bool guest)
+{
+        struct intel_gvt *gvt = spt->vgpu->gvt;
+        struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+        e->type = get_entry_type(type);
+
+        if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
+                return e;
+
+        ops->get_entry(page_table, e, index, guest,
+                        spt->guest_page.gfn << GTT_PAGE_SHIFT,
+                        spt->vgpu);
+        ops->test_pse(e);
+        return e;
+}
+
+static inline struct intel_gvt_gtt_entry *ppgtt_spt_set_entry(
+                struct intel_vgpu_ppgtt_spt *spt,
+                void *page_table, int type,
+                struct intel_gvt_gtt_entry *e, unsigned long index,
+                bool guest)
+{
+        struct intel_gvt *gvt = spt->vgpu->gvt;
+        struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+        if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
+                return e;
+
+        return ops->set_entry(page_table, e, index, guest,
+                        spt->guest_page.gfn << GTT_PAGE_SHIFT,
+                        spt->vgpu);
+}
+
+#define ppgtt_get_guest_entry(spt, e, index) \
+        ppgtt_spt_get_entry(spt, NULL, \
+                spt->guest_page_type, e, index, true)
+
+#define ppgtt_set_guest_entry(spt, e, index) \
+        ppgtt_spt_set_entry(spt, NULL, \
+                spt->guest_page_type, e, index, true)
+
+#define ppgtt_get_shadow_entry(spt, e, index) \
+        ppgtt_spt_get_entry(spt, spt->shadow_page.vaddr, \
+                spt->shadow_page.type, e, index, false)
+
+#define ppgtt_set_shadow_entry(spt, e, index) \
+        ppgtt_spt_set_entry(spt, spt->shadow_page.vaddr, \
+                spt->shadow_page.type, e, index, false)
+
+/**
+ * intel_vgpu_init_guest_page - init a guest page data structure
+ * @vgpu: a vGPU
+ * @p: a guest page data structure
+ * @gfn: guest memory page frame number
+ * @handler: function will be called when target guest memory page has
+ * been modified.
+ *
+ * This function is called when user wants to track a guest memory page.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *p,
+                unsigned long gfn,
+                int (*handler)(void *, u64, void *, int),
+                void *data)
+{
+        INIT_HLIST_NODE(&p->node);
+
+        p->writeprotection = false;
+        p->gfn = gfn;
+        p->handler = handler;
+        p->data = data;
+        p->oos_page = NULL;
+        p->write_cnt = 0;
+
+        hash_add(vgpu->gtt.guest_page_hash_table, &p->node, p->gfn);
+        return 0;
+}
+
+static int detach_oos_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_oos_page *oos_page);
+
+/**
+ * intel_vgpu_clean_guest_page - release the resource owned by guest page data
+ * structure
+ * @vgpu: a vGPU
+ * @p: a tracked guest page
+ *
+ * This function is called when user tries to stop tracking a guest memory
+ * page.
+ */
+void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *p)
+{
+        if (!hlist_unhashed(&p->node))
+                hash_del(&p->node);
+
+        if (p->oos_page)
+                detach_oos_page(vgpu, p->oos_page);
+
+        if (p->writeprotection)
+                intel_gvt_hypervisor_unset_wp_page(vgpu, p);
+}
+
+/**
+ * intel_vgpu_find_guest_page - find a guest page data structure by GFN.
+ * @vgpu: a vGPU
+ * @gfn: guest memory page frame number
+ *
+ * This function is called when emulation logic wants to know if a trapped GFN
+ * is a tracked guest page.
+ *
+ * Returns:
+ * Pointer to guest page data structure, NULL if failed.
+ */
+struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+                struct intel_vgpu *vgpu, unsigned long gfn)
+{
+        struct intel_vgpu_guest_page *p;
+
+        hash_for_each_possible(vgpu->gtt.guest_page_hash_table,
+                p, node, gfn) {
+                if (p->gfn == gfn)
+                        return p;
+        }
+        return NULL;
+}
+
+static inline int init_shadow_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_shadow_page *p, int type)
+{
+        p->vaddr = page_address(p->page);
+        p->type = type;
+
+        INIT_HLIST_NODE(&p->node);
+
+        p->mfn = intel_gvt_hypervisor_virt_to_mfn(p->vaddr);
+        if (p->mfn == INTEL_GVT_INVALID_ADDR)
+                return -EFAULT;
+
+        hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
+        return 0;
+}
+
+static inline void clean_shadow_page(struct intel_vgpu_shadow_page *p)
+{
+        if (!hlist_unhashed(&p->node))
+                hash_del(&p->node);
+}
+
+static inline struct intel_vgpu_shadow_page *find_shadow_page(
+                struct intel_vgpu *vgpu, unsigned long mfn)
+{
+        struct intel_vgpu_shadow_page *p;
+
+        hash_for_each_possible(vgpu->gtt.shadow_page_hash_table,
+                p, node, mfn) {
+                if (p->mfn == mfn)
+                        return p;
+        }
+        return NULL;
+}
+
+#define guest_page_to_ppgtt_spt(ptr) \
+        container_of(ptr, struct intel_vgpu_ppgtt_spt, guest_page)
+
+#define shadow_page_to_ppgtt_spt(ptr) \
+        container_of(ptr, struct intel_vgpu_ppgtt_spt, shadow_page)
+
+static void *alloc_spt(gfp_t gfp_mask)
+{
+        struct intel_vgpu_ppgtt_spt *spt;
+
+        spt = kzalloc(sizeof(*spt), gfp_mask);
+        if (!spt)
+                return NULL;
+
+        spt->shadow_page.page = alloc_page(gfp_mask);
+        if (!spt->shadow_page.page) {
+                kfree(spt);
+                return NULL;
+        }
+        return spt;
+}
+
+static void free_spt(struct intel_vgpu_ppgtt_spt *spt)
+{
+        __free_page(spt->shadow_page.page);
+        kfree(spt);
+}
+
+static void ppgtt_free_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+        trace_spt_free(spt->vgpu->id, spt, spt->shadow_page.type);
+
+        clean_shadow_page(&spt->shadow_page);
+        intel_vgpu_clean_guest_page(spt->vgpu, &spt->guest_page);
+        list_del_init(&spt->post_shadow_list);
+
+        free_spt(spt);
+}
+
+static void ppgtt_free_all_shadow_page(struct intel_vgpu *vgpu)
+{
+        struct hlist_node *n;
+        struct intel_vgpu_shadow_page *sp;
+        int i;
+
+        hash_for_each_safe(vgpu->gtt.shadow_page_hash_table, i, n, sp, node)
+                ppgtt_free_shadow_page(shadow_page_to_ppgtt_spt(sp));
+}
+
+static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+                u64 pa, void *p_data, int bytes);
+
+static int ppgtt_write_protection_handler(void *gp, u64 pa,
+                void *p_data, int bytes)
+{
+        struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
+        int ret;
+
+        if (bytes != 4 && bytes != 8)
+                return -EINVAL;
+
+        if (!gpt->writeprotection)
+                return -EINVAL;
+
+        ret = ppgtt_handle_guest_write_page_table_bytes(gp,
+                pa, p_data, bytes);
+        if (ret)
+                return ret;
+        return ret;
+}
+
+static int reclaim_one_mm(struct intel_gvt *gvt);
+
+static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_shadow_page(
+                struct intel_vgpu *vgpu, int type, unsigned long gfn)
+{
+        struct intel_vgpu_ppgtt_spt *spt = NULL;
+        int ret;
+
+retry:
+        spt = alloc_spt(GFP_KERNEL | __GFP_ZERO);
+        if (!spt) {
+                if (reclaim_one_mm(vgpu->gvt))
+                        goto retry;
+
+                gvt_err("fail to allocate ppgtt shadow page\n");
+                return ERR_PTR(-ENOMEM);
+        }
+
+        spt->vgpu = vgpu;
+        spt->guest_page_type = type;
+        atomic_set(&spt->refcount, 1);
+        INIT_LIST_HEAD(&spt->post_shadow_list);
+
+        /*
+         * TODO: guest page type may be different with shadow page type,
+         *       when we support PSE page in future.
+         */
+        ret = init_shadow_page(vgpu, &spt->shadow_page, type);
+        if (ret) {
+                gvt_err("fail to initialize shadow page for spt\n");
+                goto err;
+        }
+
+        ret = intel_vgpu_init_guest_page(vgpu, &spt->guest_page,
+                        gfn, ppgtt_write_protection_handler, NULL);
+        if (ret) {
+                gvt_err("fail to initialize guest page for spt\n");
+                goto err;
+        }
+
+        trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);
+        return spt;
+err:
+        ppgtt_free_shadow_page(spt);
+        return ERR_PTR(ret);
+}
+
+static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page(
+                struct intel_vgpu *vgpu, unsigned long mfn)
+{
+        struct intel_vgpu_shadow_page *p = find_shadow_page(vgpu, mfn);
+
+        if (p)
+                return shadow_page_to_ppgtt_spt(p);
+
+        gvt_err("vgpu%d: fail to find ppgtt shadow page: 0x%lx\n",
+                        vgpu->id, mfn);
+        return NULL;
+}
+
+#define pt_entry_size_shift(spt) \
+        ((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
+
+#define pt_entries(spt) \
+        (GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
+
+#define for_each_present_guest_entry(spt, e, i) \
+        for (i = 0; i < pt_entries(spt); i++) \
+        if (spt->vgpu->gvt->gtt.pte_ops->test_present( \
+                ppgtt_get_guest_entry(spt, e, i)))
+
+#define for_each_present_shadow_entry(spt, e, i) \
+        for (i = 0; i < pt_entries(spt); i++) \
+        if (spt->vgpu->gvt->gtt.pte_ops->test_present( \
+                ppgtt_get_shadow_entry(spt, e, i)))
+
+static void ppgtt_get_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+        int v = atomic_read(&spt->refcount);
+
+        trace_spt_refcount(spt->vgpu->id, "inc", spt, v, (v + 1));
+
+        atomic_inc(&spt->refcount);
+}
+
+static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt);
+
+static int ppgtt_invalidate_shadow_page_by_shadow_entry(struct intel_vgpu *vgpu,
+                struct intel_gvt_gtt_entry *e)
+{
+        struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+        struct intel_vgpu_ppgtt_spt *s;
+
+        if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(e->type))))
+                return -EINVAL;
+
+        if (ops->get_pfn(e) == vgpu->gtt.scratch_page_mfn)
+                return 0;
+
+        s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
+        if (!s) {
+                gvt_err("vgpu%d: fail to find shadow page: mfn: 0x%lx\n",
+                                vgpu->id, ops->get_pfn(e));
+                return -ENXIO;
+        }
+        return ppgtt_invalidate_shadow_page(s);
+}
+
+static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+        struct intel_gvt_gtt_entry e;
+        unsigned long index;
+        int ret;
+        int v = atomic_read(&spt->refcount);
+
+        trace_spt_change(spt->vgpu->id, "die", spt,
+                        spt->guest_page.gfn, spt->shadow_page.type);
+
+        trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
+
+        if (atomic_dec_return(&spt->refcount) > 0)
+                return 0;
+
+        if (gtt_type_is_pte_pt(spt->shadow_page.type))
+                goto release;
+
+        for_each_present_shadow_entry(spt, &e, index) {
+                if (!gtt_type_is_pt(get_next_pt_type(e.type))) {
+                        gvt_err("GVT doesn't support pse bit for now\n");
+                        return -EINVAL;
+                }
+                ret = ppgtt_invalidate_shadow_page_by_shadow_entry(
+                                spt->vgpu, &e);
+                if (ret)
+                        goto fail;
+        }
+release:
+        trace_spt_change(spt->vgpu->id, "release", spt,
+                        spt->guest_page.gfn, spt->shadow_page.type);
+        ppgtt_free_shadow_page(spt);
+        return 0;
+fail:
+        gvt_err("vgpu%d: fail: shadow page %p shadow entry 0x%llx type %d\n",
+                        spt->vgpu->id, spt, e.val64, e.type);
+        return ret;
+}
+
+static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt);
+
+static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
+                struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we)
+{
+        struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+        struct intel_vgpu_ppgtt_spt *s = NULL;
+        struct intel_vgpu_guest_page *g;
+        int ret;
+
+        if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(we->type)))) {
+                ret = -EINVAL;
+                goto fail;
+        }
+
+        g = intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we));
+        if (g) {
+                s = guest_page_to_ppgtt_spt(g);
+                ppgtt_get_shadow_page(s);
+        } else {
+                int type = get_next_pt_type(we->type);
+
+                s = ppgtt_alloc_shadow_page(vgpu, type, ops->get_pfn(we));
+                if (IS_ERR(s)) {
+                        ret = PTR_ERR(s);
+                        goto fail;
+                }
+
+                ret = intel_gvt_hypervisor_set_wp_page(vgpu, &s->guest_page);
+                if (ret)
+                        goto fail;
+
+                ret = ppgtt_populate_shadow_page(s);
+                if (ret)
+                        goto fail;
+
+                trace_spt_change(vgpu->id, "new", s, s->guest_page.gfn,
+                        s->shadow_page.type);
+        }
+        return s;
+fail:
+        gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n",
+                        vgpu->id, s, we->val64, we->type);
+        return ERR_PTR(ret);
+}
+
+static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
+                struct intel_vgpu_ppgtt_spt *s, struct intel_gvt_gtt_entry *ge)
+{
+        struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;
+
+        se->type = ge->type;
+        se->val64 = ge->val64;
+
+        ops->set_pfn(se, s->shadow_page.mfn);
+}
+
+static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+        struct intel_vgpu *vgpu = spt->vgpu;
+        struct intel_vgpu_ppgtt_spt *s;
+        struct intel_gvt_gtt_entry se, ge;
+        unsigned long i;
+        int ret;
+
+        trace_spt_change(spt->vgpu->id, "born", spt,
+                        spt->guest_page.gfn, spt->shadow_page.type);
+
+        if (gtt_type_is_pte_pt(spt->shadow_page.type)) {
+                for_each_present_guest_entry(spt, &ge, i) {
+                        ret = gtt_entry_p2m(vgpu, &ge, &se);
+                        if (ret)
+                                goto fail;
+                        ppgtt_set_shadow_entry(spt, &se, i);
+                }
+                return 0;
+        }
+
+        for_each_present_guest_entry(spt, &ge, i) {
+                if (!gtt_type_is_pt(get_next_pt_type(ge.type))) {
+                        gvt_err("GVT doesn't support pse bit now\n");
+                        ret = -EINVAL;
+                        goto fail;
+                }
+
+                s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge);
+                if (IS_ERR(s)) {
+                        ret = PTR_ERR(s);
+                        goto fail;
+                }
+                ppgtt_get_shadow_entry(spt, &se, i);
+                ppgtt_generate_shadow_entry(&se, s, &ge);
+                ppgtt_set_shadow_entry(spt, &se, i);
+        }
+        return 0;
+fail:
+        gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n",
+                        vgpu->id, spt, ge.val64, ge.type);
+        return ret;
+}
+
+static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt,
+                struct intel_gvt_gtt_entry *we, unsigned long index)
+{
+        struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+        struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
+        struct intel_vgpu *vgpu = spt->vgpu;
+        struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+        struct intel_gvt_gtt_entry e;
+        int ret;
+
+        trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type,
+                we->val64, index);
+
+        ppgtt_get_shadow_entry(spt, &e, index);
+        if (!ops->test_present(&e))
+                return 0;
+
+        if (ops->get_pfn(&e) == vgpu->gtt.scratch_page_mfn)
+                return 0;
+
+        if (gtt_type_is_pt(get_next_pt_type(we->type))) {
+                struct intel_vgpu_guest_page *g =
+                        intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we));
+                if (!g) {
+                        gvt_err("fail to find guest page\n");
+                        ret = -ENXIO;
+                        goto fail;
+                }
+                ret = ppgtt_invalidate_shadow_page(guest_page_to_ppgtt_spt(g));
+                if (ret)
+                        goto fail;
+        }
+        ops->set_pfn(&e, vgpu->gtt.scratch_page_mfn);
+        ppgtt_set_shadow_entry(spt, &e, index);
+        return 0;
+fail:
+        gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n",
+                        vgpu->id, spt, we->val64, we->type);
+        return ret;
+}
+
+static int ppgtt_handle_guest_entry_add(struct intel_vgpu_guest_page *gpt,
+                struct intel_gvt_gtt_entry *we, unsigned long index)
+{
+        struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+        struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
+        struct intel_vgpu *vgpu = spt->vgpu;
+        struct intel_gvt_gtt_entry m;
+        struct intel_vgpu_ppgtt_spt *s;
+        int ret;
+
+        trace_gpt_change(spt->vgpu->id, "add", spt, sp->type,
+                we->val64, index);
+
+        if (gtt_type_is_pt(get_next_pt_type(we->type))) {
+                s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, we);
+                if (IS_ERR(s)) {
+                        ret = PTR_ERR(s);
+                        goto fail;
+                }
+                ppgtt_get_shadow_entry(spt, &m, index);
+                ppgtt_generate_shadow_entry(&m, s, we);
+                ppgtt_set_shadow_entry(spt, &m, index);
+        } else {
+                ret = gtt_entry_p2m(vgpu, we, &m);
+                if (ret)
+                        goto fail;
+                ppgtt_set_shadow_entry(spt, &m, index);
+        }
+        return 0;
+fail:
+        gvt_err("vgpu%d: fail: spt %p guest entry 0x%llx type %d\n", vgpu->id,
+                        spt, we->val64, we->type);
+        return ret;
+}
+
+static int sync_oos_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_oos_page *oos_page)
+{
+        const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+        struct intel_vgpu_ppgtt_spt *spt =
+                guest_page_to_ppgtt_spt(oos_page->guest_page);
+        struct intel_gvt_gtt_entry old, new, m;
+        int index;
+        int ret;
+
+        trace_oos_change(vgpu->id, "sync", oos_page->id,
+                        oos_page->guest_page, spt->guest_page_type);
+
+        old.type = new.type = get_entry_type(spt->guest_page_type);
+        old.val64 = new.val64 = 0;
+
+        for (index = 0; index < (GTT_PAGE_SIZE >> info->gtt_entry_size_shift);
+                index++) {
+                ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu);
+                ops->get_entry(NULL, &new, index, true,
+                        oos_page->guest_page->gfn << PAGE_SHIFT, vgpu);
+
+                if (old.val64 == new.val64
+                        && !test_and_clear_bit(index, spt->post_shadow_bitmap))
+                        continue;
+
+                trace_oos_sync(vgpu->id, oos_page->id,
+                                oos_page->guest_page, spt->guest_page_type,
+                                new.val64, index);
+
+                ret = gtt_entry_p2m(vgpu, &new, &m);
+                if (ret)
+                        return ret;
+
+                ops->set_entry(oos_page->mem, &new, index, false, 0, vgpu);
+                ppgtt_set_shadow_entry(spt, &m, index);
+        }
+
+        oos_page->guest_page->write_cnt = 0;
+        list_del_init(&spt->post_shadow_list);
+        return 0;
+}
+
+static int detach_oos_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_oos_page *oos_page)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_vgpu_ppgtt_spt *spt =
+                guest_page_to_ppgtt_spt(oos_page->guest_page);
+
+        trace_oos_change(vgpu->id, "detach", oos_page->id,
+                        oos_page->guest_page, spt->guest_page_type);
+
+        oos_page->guest_page->write_cnt = 0;
+        oos_page->guest_page->oos_page = NULL;
+        oos_page->guest_page = NULL;
+
+        list_del_init(&oos_page->vm_list);
+        list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);
+
+        return 0;
+}
+
+static int attach_oos_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_oos_page *oos_page,
+                struct intel_vgpu_guest_page *gpt)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        int ret;
+
+        ret = intel_gvt_hypervisor_read_gpa(vgpu, gpt->gfn << GTT_PAGE_SHIFT,
+                oos_page->mem, GTT_PAGE_SIZE);
+        if (ret)
+                return ret;
+
+        oos_page->guest_page = gpt;
+        gpt->oos_page = oos_page;
+
+        list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);
+
+        trace_oos_change(vgpu->id, "attach", gpt->oos_page->id,
+                        gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
+        return 0;
+}
+
+static int ppgtt_set_guest_page_sync(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *gpt)
+{
+        int ret;
+
+        ret = intel_gvt_hypervisor_set_wp_page(vgpu, gpt);
+        if (ret)
+                return ret;
+
+        trace_oos_change(vgpu->id, "set page sync", gpt->oos_page->id,
+                        gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
+
+        list_del_init(&gpt->oos_page->vm_list);
+        return sync_oos_page(vgpu, gpt->oos_page);
+}
+
+static int ppgtt_allocate_oos_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *gpt)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_gtt *gtt = &gvt->gtt;
+        struct intel_vgpu_oos_page *oos_page = gpt->oos_page;
+        int ret;
+
+        WARN(oos_page, "shadow PPGTT page has already has a oos page\n");
+
+        if (list_empty(&gtt->oos_page_free_list_head)) {
+                oos_page = container_of(gtt->oos_page_use_list_head.next,
+                        struct intel_vgpu_oos_page, list);
+                ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page);
+                if (ret)
+                        return ret;
+                ret = detach_oos_page(vgpu, oos_page);
+                if (ret)
+                        return ret;
+        } else
+                oos_page = container_of(gtt->oos_page_free_list_head.next,
+                        struct intel_vgpu_oos_page, list);
+        return attach_oos_page(vgpu, oos_page, gpt);
+}
+
+static int ppgtt_set_guest_page_oos(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *gpt)
+{
+        struct intel_vgpu_oos_page *oos_page = gpt->oos_page;
+
+        if (WARN(!oos_page, "shadow PPGTT page should have a oos page\n"))
+                return -EINVAL;
+
+        trace_oos_change(vgpu->id, "set page out of sync", gpt->oos_page->id,
+                        gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
+
+        list_add_tail(&oos_page->vm_list, &vgpu->gtt.oos_page_list_head);
+        return intel_gvt_hypervisor_unset_wp_page(vgpu, gpt);
+}
+
+/**
+ * intel_vgpu_sync_oos_pages - sync all the out-of-synced shadow for vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is called before submitting a guest workload to host,
+ * to sync all the out-of-synced shadow for vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu)
+{
+        struct list_head *pos, *n;
+        struct intel_vgpu_oos_page *oos_page;
+        int ret;
+
+        if (!enable_out_of_sync)
+                return 0;
+
+        list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {
+                oos_page = container_of(pos,
+                                struct intel_vgpu_oos_page, vm_list);
+                ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page);
+                if (ret)
+                        return ret;
+        }
+        return 0;
+}
+
+/*
+ * The heart of PPGTT shadow page table.
+ */
+static int ppgtt_handle_guest_write_page_table(
+                struct intel_vgpu_guest_page *gpt,
+                struct intel_gvt_gtt_entry *we, unsigned long index)
+{
+        struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+        struct intel_vgpu *vgpu = spt->vgpu;
+        struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+        struct intel_gvt_gtt_entry ge;
+
+        int old_present, new_present;
+        int ret;
+
+        ppgtt_get_guest_entry(spt, &ge, index);
+
+        old_present = ops->test_present(&ge);
+        new_present = ops->test_present(we);
+
+        ppgtt_set_guest_entry(spt, we, index);
+
+        if (old_present) {
+                ret = ppgtt_handle_guest_entry_removal(gpt, &ge, index);
+                if (ret)
+                        goto fail;
+        }
+        if (new_present) {
+                ret = ppgtt_handle_guest_entry_add(gpt, we, index);
+                if (ret)
+                        goto fail;
+        }
+        return 0;
+fail:
+        gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d.\n",
+                        vgpu->id, spt, we->val64, we->type);
+        return ret;
+}
+
+static inline bool can_do_out_of_sync(struct intel_vgpu_guest_page *gpt)
+{
+        return enable_out_of_sync
+                && gtt_type_is_pte_pt(
+                        guest_page_to_ppgtt_spt(gpt)->guest_page_type)
+                && gpt->write_cnt >= 2;
+}
+
+static void ppgtt_set_post_shadow(struct intel_vgpu_ppgtt_spt *spt,
+                unsigned long index)
+{
+        set_bit(index, spt->post_shadow_bitmap);
+        if (!list_empty(&spt->post_shadow_list))
+                return;
+
+        list_add_tail(&spt->post_shadow_list,
+                        &spt->vgpu->gtt.post_shadow_list_head);
+}
+
+/**
+ * intel_vgpu_flush_post_shadow - flush the post shadow transactions
+ * @vgpu: a vGPU
+ *
+ * This function is called before submitting a guest workload to host,
+ * to flush all the post shadows for a vGPU.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
+{
+        struct list_head *pos, *n;
+        struct intel_vgpu_ppgtt_spt *spt;
+        struct intel_gvt_gtt_entry ge, e;
+        unsigned long index;
+        int ret;
+
+        list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {
+                spt = container_of(pos, struct intel_vgpu_ppgtt_spt,
+                                post_shadow_list);
+
+                for_each_set_bit(index, spt->post_shadow_bitmap,
+                                GTT_ENTRY_NUM_IN_ONE_PAGE) {
+                        ppgtt_get_guest_entry(spt, &ge, index);
+                        e = ge;
+                        e.val64 = 0;
+                        ppgtt_set_guest_entry(spt, &e, index);
+
+                        ret = ppgtt_handle_guest_write_page_table(
+                                        &spt->guest_page, &ge, index);
+                        if (ret)
+                                return ret;
+                        clear_bit(index, spt->post_shadow_bitmap);
+                }
+                list_del_init(&spt->post_shadow_list);
+        }
+        return 0;
+}
+
+static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+                u64 pa, void *p_data, int bytes)
+{
+        struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
+        struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+        struct intel_vgpu *vgpu = spt->vgpu;
+        struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+        const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+        struct intel_gvt_gtt_entry we;
+        unsigned long index;
+        int ret;
+
+        index = (pa & (PAGE_SIZE - 1)) >> info->gtt_entry_size_shift;
+
+        ppgtt_get_guest_entry(spt, &we, index);
+        memcpy((void *)&we.val64 + (pa & (info->gtt_entry_size - 1)),
+                        p_data, bytes);
+
+        ops->test_pse(&we);
+
+        if (bytes == info->gtt_entry_size) {
+                ret = ppgtt_handle_guest_write_page_table(gpt, &we, index);
+                if (ret)
+                        return ret;
+        } else {
+                struct intel_gvt_gtt_entry ge;
+
+                ppgtt_get_guest_entry(spt, &ge, index);
+
+                if (!test_bit(index, spt->post_shadow_bitmap)) {
+                        ret = ppgtt_handle_guest_entry_removal(gpt,
+                                        &ge, index);
+                        if (ret)
+                                return ret;
+                }
+
+                ppgtt_set_post_shadow(spt, index);
+                ppgtt_set_guest_entry(spt, &we, index);
+        }
+
+        if (!enable_out_of_sync)
+                return 0;
+
+        gpt->write_cnt++;
+
+        if (gpt->oos_page)
+                ops->set_entry(gpt->oos_page->mem, &we, index,
+                                false, 0, vgpu);
+
+        if (can_do_out_of_sync(gpt)) {
+                if (!gpt->oos_page)
+                        ppgtt_allocate_oos_page(vgpu, gpt);
+
+                ret = ppgtt_set_guest_page_oos(vgpu, gpt);
+                if (ret < 0)
+                        return ret;
+        }
+        return 0;
+}
+
+/*
+ * mm page table allocation policy for bdw+
+ *  - for ggtt, only virtual page table will be allocated.
+ *  - for ppgtt, dedicated virtual/shadow page table will be allocated.
+ */
+static int gen8_mm_alloc_page_table(struct intel_vgpu_mm *mm)
+{
+        struct intel_vgpu *vgpu = mm->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        const struct intel_gvt_device_info *info = &gvt->device_info;
+        void *mem;
+
+        if (mm->type == INTEL_GVT_MM_PPGTT) {
+                mm->page_table_entry_cnt = 4;
+                mm->page_table_entry_size = mm->page_table_entry_cnt *
+                        info->gtt_entry_size;
+                mem = kzalloc(mm->has_shadow_page_table ?
+                        mm->page_table_entry_size * 2
+                                : mm->page_table_entry_size,
+                        GFP_ATOMIC);
+                if (!mem)
+                        return -ENOMEM;
+                mm->virtual_page_table = mem;
+                if (!mm->has_shadow_page_table)
+                        return 0;
+                mm->shadow_page_table = mem + mm->page_table_entry_size;
+        } else if (mm->type == INTEL_GVT_MM_GGTT) {
+                mm->page_table_entry_cnt =
+                        (gvt_ggtt_gm_sz(gvt) >> GTT_PAGE_SHIFT);
+                mm->page_table_entry_size = mm->page_table_entry_cnt *
+                        info->gtt_entry_size;
+                mem = vzalloc(mm->page_table_entry_size);
+                if (!mem)
+                        return -ENOMEM;
+                mm->virtual_page_table = mem;
+        }
+        return 0;
+}
+
+static void gen8_mm_free_page_table(struct intel_vgpu_mm *mm)
+{
+        if (mm->type == INTEL_GVT_MM_PPGTT) {
+                kfree(mm->virtual_page_table);
+        } else if (mm->type == INTEL_GVT_MM_GGTT) {
+                if (mm->virtual_page_table)
+                        vfree(mm->virtual_page_table);
+        }
+        mm->virtual_page_table = mm->shadow_page_table = NULL;
+}
+
+static void invalidate_mm(struct intel_vgpu_mm *mm)
+{
+        struct intel_vgpu *vgpu = mm->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_gtt *gtt = &gvt->gtt;
+        struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
+        struct intel_gvt_gtt_entry se;
+        int i;
+
+        if (WARN_ON(!mm->has_shadow_page_table || !mm->shadowed))
+                return;
+
+        for (i = 0; i < mm->page_table_entry_cnt; i++) {
+                ppgtt_get_shadow_root_entry(mm, &se, i);
+                if (!ops->test_present(&se))
+                        continue;
+                ppgtt_invalidate_shadow_page_by_shadow_entry(
+                                vgpu, &se);
+                se.val64 = 0;
+                ppgtt_set_shadow_root_entry(mm, &se, i);
+
+                trace_gpt_change(vgpu->id, "destroy root pointer",
+                                NULL, se.type, se.val64, i);
+        }
+        mm->shadowed = false;
+}
+
+/**
+ * intel_vgpu_destroy_mm - destroy a mm object
+ * @mm: a kref object
+ *
+ * This function is used to destroy a mm object for vGPU
+ *
+ */
+void intel_vgpu_destroy_mm(struct kref *mm_ref)
+{
+        struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref);
+        struct intel_vgpu *vgpu = mm->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_gtt *gtt = &gvt->gtt;
+
+        if (!mm->initialized)
+                goto out;
+
+        list_del(&mm->list);
+        list_del(&mm->lru_list);
+
+        if (mm->has_shadow_page_table)
+                invalidate_mm(mm);
+
+        gtt->mm_free_page_table(mm);
+out:
+        kfree(mm);
+}
+
+static int shadow_mm(struct intel_vgpu_mm *mm)
+{
+        struct intel_vgpu *vgpu = mm->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_gtt *gtt = &gvt->gtt;
+        struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
+        struct intel_vgpu_ppgtt_spt *spt;
+        struct intel_gvt_gtt_entry ge, se;
+        int i;
+        int ret;
+
+        if (WARN_ON(!mm->has_shadow_page_table || mm->shadowed))
+                return 0;
+
+        mm->shadowed = true;
+
+        for (i = 0; i < mm->page_table_entry_cnt; i++) {
+                ppgtt_get_guest_root_entry(mm, &ge, i);
+                if (!ops->test_present(&ge))
+                        continue;
+
+                trace_gpt_change(vgpu->id, __func__, NULL,
+                                ge.type, ge.val64, i);
+
+                spt = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge);
+                if (IS_ERR(spt)) {
+                        gvt_err("fail to populate guest root pointer\n");
+                        ret = PTR_ERR(spt);
+                        goto fail;
+                }
+                ppgtt_generate_shadow_entry(&se, spt, &ge);
+                ppgtt_set_shadow_root_entry(mm, &se, i);
+
+                trace_gpt_change(vgpu->id, "populate root pointer",
+                                NULL, se.type, se.val64, i);
+        }
+        return 0;
+fail:
+        invalidate_mm(mm);
+        return ret;
+}
+
+/**
+ * intel_vgpu_create_mm - create a mm object for a vGPU
+ * @vgpu: a vGPU
+ * @mm_type: mm object type, should be PPGTT or GGTT
+ * @virtual_page_table: page table root pointers. Could be NULL if user wants
+ *      to populate shadow later.
+ * @page_table_level: describe the page table level of the mm object
+ * @pde_base_index: pde root pointer base in GGTT MMIO.
+ *
+ * This function is used to create a mm object for a vGPU.
+ *
+ * Returns:
+ * Zero on success, negative error code in pointer if failed.
+ */
+struct intel_vgpu_mm *intel_vgpu_create_mm(struct intel_vgpu *vgpu,
+                int mm_type, void *virtual_page_table, int page_table_level,
+                u32 pde_base_index)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_gtt *gtt = &gvt->gtt;
+        struct intel_vgpu_mm *mm;
+        int ret;
+
+        mm = kzalloc(sizeof(*mm), GFP_ATOMIC);
+        if (!mm) {
+                ret = -ENOMEM;
+                goto fail;
+        }
+
+        mm->type = mm_type;
+
+        if (page_table_level == 1)
+                mm->page_table_entry_type = GTT_TYPE_GGTT_PTE;
+        else if (page_table_level == 3)
+                mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
+        else if (page_table_level == 4)
+                mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
+        else {
+                WARN_ON(1);
+                ret = -EINVAL;
+                goto fail;
+        }
+
+        mm->page_table_level = page_table_level;
+        mm->pde_base_index = pde_base_index;
+
+        mm->vgpu = vgpu;
+        mm->has_shadow_page_table = !!(mm_type == INTEL_GVT_MM_PPGTT);
+
+        kref_init(&mm->ref);
+        atomic_set(&mm->pincount, 0);
+        INIT_LIST_HEAD(&mm->list);
+        INIT_LIST_HEAD(&mm->lru_list);
+        list_add_tail(&mm->list, &vgpu->gtt.mm_list_head);
+
+        ret = gtt->mm_alloc_page_table(mm);
+        if (ret) {
+                gvt_err("fail to allocate page table for mm\n");
+                goto fail;
+        }
+
+        mm->initialized = true;
+
+        if (virtual_page_table)
+                memcpy(mm->virtual_page_table, virtual_page_table,
+                                mm->page_table_entry_size);
+
+        if (mm->has_shadow_page_table) {
+                ret = shadow_mm(mm);
+                if (ret)
+                        goto fail;
+                list_add_tail(&mm->lru_list, &gvt->gtt.mm_lru_list_head);
+        }
+        return mm;
+fail:
+        gvt_err("fail to create mm\n");
+        if (mm)
+                intel_gvt_mm_unreference(mm);
+        return ERR_PTR(ret);
+}
+
+/**
+ * intel_vgpu_unpin_mm - decrease the pin count of a vGPU mm object
+ * @mm: a vGPU mm object
+ *
+ * This function is called when user doesn't want to use a vGPU mm object
+ */
+void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm)
+{
+        if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
+                return;
+
+        atomic_dec(&mm->pincount);
+}
+
+/**
+ * intel_vgpu_pin_mm - increase the pin count of a vGPU mm object
+ * @vgpu: a vGPU
+ *
+ * This function is called when user wants to use a vGPU mm object. If this
+ * mm object hasn't been shadowed yet, the shadow will be populated at this
+ * time.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm)
+{
+        int ret;
+
+        if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
+                return 0;
+
+        atomic_inc(&mm->pincount);
+
+        if (!mm->shadowed) {
+                ret = shadow_mm(mm);
+                if (ret)
+                        return ret;
+        }
+
+        list_del_init(&mm->lru_list);
+        list_add_tail(&mm->lru_list, &mm->vgpu->gvt->gtt.mm_lru_list_head);
+        return 0;
+}
+
+static int reclaim_one_mm(struct intel_gvt *gvt)
+{
+        struct intel_vgpu_mm *mm;
+        struct list_head *pos, *n;
+
+        list_for_each_safe(pos, n, &gvt->gtt.mm_lru_list_head) {
+                mm = container_of(pos, struct intel_vgpu_mm, lru_list);
+
+                if (mm->type != INTEL_GVT_MM_PPGTT)
+                        continue;
+                if (atomic_read(&mm->pincount))
+                        continue;
+
+                list_del_init(&mm->lru_list);
+                invalidate_mm(mm);
+                return 1;
+        }
+        return 0;
+}
+
+/*
+ * GMA translation APIs.
+ */
+static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm,
+                struct intel_gvt_gtt_entry *e, unsigned long index, bool guest)
+{
+        struct intel_vgpu *vgpu = mm->vgpu;
+        struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+        struct intel_vgpu_ppgtt_spt *s;
+
+        if (WARN_ON(!mm->has_shadow_page_table))
+                return -EINVAL;
+
+        s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
+        if (!s)
+                return -ENXIO;
+
+        if (!guest)
+                ppgtt_get_shadow_entry(s, e, index);
+        else
+                ppgtt_get_guest_entry(s, e, index);
+        return 0;
+}
+
+/**
+ * intel_vgpu_gma_to_gpa - translate a gma to GPA
+ * @mm: mm object. could be a PPGTT or GGTT mm object
+ * @gma: graphics memory address in this mm object
+ *
+ * This function is used to translate a graphics memory address in specific
+ * graphics memory space to guest physical address.
+ *
+ * Returns:
+ * Guest physical address on success, INTEL_GVT_INVALID_ADDR if failed.
+ */
+unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
+{
+        struct intel_vgpu *vgpu = mm->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;
+        struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;
+        unsigned long gpa = INTEL_GVT_INVALID_ADDR;
+        unsigned long gma_index[4];
+        struct intel_gvt_gtt_entry e;
+        int i, index;
+        int ret;
+
+        if (mm->type != INTEL_GVT_MM_GGTT && mm->type != INTEL_GVT_MM_PPGTT)
+                return INTEL_GVT_INVALID_ADDR;
+
+        if (mm->type == INTEL_GVT_MM_GGTT) {
+                if (!vgpu_gmadr_is_valid(vgpu, gma))
+                        goto err;
+
+                ggtt_get_guest_entry(mm, &e,
+                        gma_ops->gma_to_ggtt_pte_index(gma));
+                gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
+                        + (gma & ~GTT_PAGE_MASK);
+
+                trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa);
+                return gpa;
+        }
+
+        switch (mm->page_table_level) {
+        case 4:
+                ppgtt_get_shadow_root_entry(mm, &e, 0);
+                gma_index[0] = gma_ops->gma_to_pml4_index(gma);
+                gma_index[1] = gma_ops->gma_to_l4_pdp_index(gma);
+                gma_index[2] = gma_ops->gma_to_pde_index(gma);
+                gma_index[3] = gma_ops->gma_to_pte_index(gma);
+                index = 4;
+                break;
+        case 3:
+                ppgtt_get_shadow_root_entry(mm, &e,
+                                gma_ops->gma_to_l3_pdp_index(gma));
+                gma_index[0] = gma_ops->gma_to_pde_index(gma);
+                gma_index[1] = gma_ops->gma_to_pte_index(gma);
+                index = 2;
+                break;
+        case 2:
+                ppgtt_get_shadow_root_entry(mm, &e,
+                                gma_ops->gma_to_pde_index(gma));
+                gma_index[0] = gma_ops->gma_to_pte_index(gma);
+                index = 1;
+                break;
+        default:
+                WARN_ON(1);
+                goto err;
+        }
+
+        /* walk into the shadow page table and get gpa from guest entry */
+        for (i = 0; i < index; i++) {
+                ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i],
+                        (i == index - 1));
+                if (ret)
+                        goto err;
+        }
+
+        gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
+                + (gma & ~GTT_PAGE_MASK);
+
+        trace_gma_translate(vgpu->id, "ppgtt", 0,
+                        mm->page_table_level, gma, gpa);
+        return gpa;
+err:
+        gvt_err("invalid mm type: %d gma %lx\n", mm->type, gma);
+        return INTEL_GVT_INVALID_ADDR;
+}
+
+static int emulate_gtt_mmio_read(struct intel_vgpu *vgpu,
+        unsigned int off, void *p_data, unsigned int bytes)
+{
+        struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
+        const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+        unsigned long index = off >> info->gtt_entry_size_shift;
+        struct intel_gvt_gtt_entry e;
+
+        if (bytes != 4 && bytes != 8)
+                return -EINVAL;
+
+        ggtt_get_guest_entry(ggtt_mm, &e, index);
+        memcpy(p_data, (void *)&e.val64 + (off & (info->gtt_entry_size - 1)),
+                        bytes);
+        return 0;
+}
+
+/**
+ * intel_vgpu_emulate_gtt_mmio_read - emulate GTT MMIO register read
+ * @vgpu: a vGPU
+ * @off: register offset
+ * @p_data: data will be returned to guest
+ * @bytes: data length
+ *
+ * This function is used to emulate the GTT MMIO register read
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
+        void *p_data, unsigned int bytes)
+{
+        const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+        int ret;
+
+        if (bytes != 4 && bytes != 8)
+                return -EINVAL;
+
+        off -= info->gtt_start_offset;
+        ret = emulate_gtt_mmio_read(vgpu, off, p_data, bytes);
+        return ret;
+}
+
+static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
+        void *p_data, unsigned int bytes)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        const struct intel_gvt_device_info *info = &gvt->device_info;
+        struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
+        struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+        unsigned long g_gtt_index = off >> info->gtt_entry_size_shift;
+        unsigned long gma;
+        struct intel_gvt_gtt_entry e, m;
+        int ret;
+
+        if (bytes != 4 && bytes != 8)
+                return -EINVAL;
+
+        gma = g_gtt_index << GTT_PAGE_SHIFT;
+
+        /* the VM may configure the whole GM space when ballooning is used */
+        if (WARN_ONCE(!vgpu_gmadr_is_valid(vgpu, gma),
+                                "vgpu%d: found oob ggtt write, offset %x\n",
+                                vgpu->id, off)) {
+                return 0;
+        }
+
+        ggtt_get_guest_entry(ggtt_mm, &e, g_gtt_index);
+
+        memcpy((void *)&e.val64 + (off & (info->gtt_entry_size - 1)), p_data,
+                        bytes);
+
+        if (ops->test_present(&e)) {
+                ret = gtt_entry_p2m(vgpu, &e, &m);
+                if (ret) {
+                        gvt_err("vgpu%d: fail to translate guest gtt entry\n",
+                                        vgpu->id);
+                        return ret;
+                }
+        } else {
+                m = e;
+                m.val64 = 0;
+        }
+
+        ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index);
+        ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index);
+        return 0;
+}
+
+/*
+ * intel_vgpu_emulate_gtt_mmio_write - emulate GTT MMIO register write
+ * @vgpu: a vGPU
+ * @off: register offset
+ * @p_data: data from guest write
+ * @bytes: data length
+ *
+ * This function is used to emulate the GTT MMIO register write
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
+        void *p_data, unsigned int bytes)
+{
+        const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+        int ret;
+
+        if (bytes != 4 && bytes != 8)
+                return -EINVAL;
+
+        off -= info->gtt_start_offset;
+        ret = emulate_gtt_mmio_write(vgpu, off, p_data, bytes);
+        return ret;
+}
+
+static int create_scratch_page(struct intel_vgpu *vgpu)
+{
+        struct intel_vgpu_gtt *gtt = &vgpu->gtt;
+        void *p;
+        void *vaddr;
+        unsigned long mfn;
+
+        gtt->scratch_page = alloc_page(GFP_KERNEL);
+        if (!gtt->scratch_page) {
+                gvt_err("Failed to allocate scratch page.\n");
+                return -ENOMEM;
+        }
+
+        /* set to zero */
+        p = kmap_atomic(gtt->scratch_page);
+        memset(p, 0, PAGE_SIZE);
+        kunmap_atomic(p);
+
+        /* translate page to mfn */
+        vaddr = page_address(gtt->scratch_page);
+        mfn = intel_gvt_hypervisor_virt_to_mfn(vaddr);
+
+        if (mfn == INTEL_GVT_INVALID_ADDR) {
+                gvt_err("fail to translate vaddr:0x%llx\n", (u64)vaddr);
+                __free_page(gtt->scratch_page);
+                gtt->scratch_page = NULL;
+                return -ENXIO;
+        }
+
+        gtt->scratch_page_mfn = mfn;
+        gvt_dbg_core("vgpu%d create scratch page: mfn=0x%lx\n", vgpu->id, mfn);
+        return 0;
+}
+
+static void release_scratch_page(struct intel_vgpu *vgpu)
+{
+        if (vgpu->gtt.scratch_page != NULL) {
+                __free_page(vgpu->gtt.scratch_page);
+                vgpu->gtt.scratch_page = NULL;
+                vgpu->gtt.scratch_page_mfn = 0;
+        }
+}
+
+/**
+ * intel_vgpu_init_gtt - initialize per-vGPU graphics memory virulization
+ * @vgpu: a vGPU
+ *
+ * This function is used to initialize per-vGPU graphics memory virtualization
+ * components.
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
+{
+        struct intel_vgpu_gtt *gtt = &vgpu->gtt;
+        struct intel_vgpu_mm *ggtt_mm;
+
+        hash_init(gtt->guest_page_hash_table);
+        hash_init(gtt->shadow_page_hash_table);
+
+        INIT_LIST_HEAD(&gtt->mm_list_head);
+        INIT_LIST_HEAD(&gtt->oos_page_list_head);
+        INIT_LIST_HEAD(&gtt->post_shadow_list_head);
+
+        ggtt_mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_GGTT,
+                        NULL, 1, 0);
+        if (IS_ERR(ggtt_mm)) {
+                gvt_err("fail to create mm for ggtt.\n");
+                return PTR_ERR(ggtt_mm);
+        }
+
+        gtt->ggtt_mm = ggtt_mm;
+
+        return create_scratch_page(vgpu);
+}
+
+/**
+ * intel_vgpu_clean_gtt - clean up per-vGPU graphics memory virulization
+ * @vgpu: a vGPU
+ *
+ * This function is used to clean up per-vGPU graphics memory virtualization
+ * components.
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu)
+{
+        struct list_head *pos, *n;
+        struct intel_vgpu_mm *mm;
+
+        ppgtt_free_all_shadow_page(vgpu);
+        release_scratch_page(vgpu);
+
+        list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) {
+                mm = container_of(pos, struct intel_vgpu_mm, list);
+                vgpu->gvt->gtt.mm_free_page_table(mm);
+                list_del(&mm->list);
+                list_del(&mm->lru_list);
+                kfree(mm);
+        }
+}
+
+static void clean_spt_oos(struct intel_gvt *gvt)
+{
+        struct intel_gvt_gtt *gtt = &gvt->gtt;
+        struct list_head *pos, *n;
+        struct intel_vgpu_oos_page *oos_page;
+
+        WARN(!list_empty(&gtt->oos_page_use_list_head),
+                "someone is still using oos page\n");
+
+        list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {
+                oos_page = container_of(pos, struct intel_vgpu_oos_page, list);
+                list_del(&oos_page->list);
+                kfree(oos_page);
+        }
+}
+
+static int setup_spt_oos(struct intel_gvt *gvt)
+{
+        struct intel_gvt_gtt *gtt = &gvt->gtt;
+        struct intel_vgpu_oos_page *oos_page;
+        int i;
+        int ret;
+
+        INIT_LIST_HEAD(&gtt->oos_page_free_list_head);
+        INIT_LIST_HEAD(&gtt->oos_page_use_list_head);
+
+        for (i = 0; i < preallocated_oos_pages; i++) {
+                oos_page = kzalloc(sizeof(*oos_page), GFP_KERNEL);
+                if (!oos_page) {
+                        gvt_err("fail to pre-allocate oos page\n");
+                        ret = -ENOMEM;
+                        goto fail;
+                }
+
+                INIT_LIST_HEAD(&oos_page->list);
+                INIT_LIST_HEAD(&oos_page->vm_list);
+                oos_page->id = i;
+                list_add_tail(&oos_page->list, &gtt->oos_page_free_list_head);
+        }
+
+        gvt_dbg_mm("%d oos pages preallocated\n", i);
+
+        return 0;
+fail:
+        clean_spt_oos(gvt);
+        return ret;
+}
+
+/**
+ * intel_vgpu_find_ppgtt_mm - find a PPGTT mm object
+ * @vgpu: a vGPU
+ * @page_table_level: PPGTT page table level
+ * @root_entry: PPGTT page table root pointers
+ *
+ * This function is used to find a PPGTT mm object from mm object pool
+ *
+ * Returns:
+ * pointer to mm object on success, NULL if failed.
+ */
+struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
+                int page_table_level, void *root_entry)
+{
+        struct list_head *pos;
+        struct intel_vgpu_mm *mm;
+        u64 *src, *dst;
+
+        list_for_each(pos, &vgpu->gtt.mm_list_head) {
+                mm = container_of(pos, struct intel_vgpu_mm, list);
+                if (mm->type != INTEL_GVT_MM_PPGTT)
+                        continue;
+
+                if (mm->page_table_level != page_table_level)
+                        continue;
+
+                src = root_entry;
+                dst = mm->virtual_page_table;
+
+                if (page_table_level == 3) {
+                        if (src[0] == dst[0]
+                                        && src[1] == dst[1]
+                                        && src[2] == dst[2]
+                                        && src[3] == dst[3])
+                                return mm;
+                } else {
+                        if (src[0] == dst[0])
+                                return mm;
+                }
+        }
+        return NULL;
+}
+
+/**
+ * intel_vgpu_g2v_create_ppgtt_mm - create a PPGTT mm object from
+ * g2v notification
+ * @vgpu: a vGPU
+ * @page_table_level: PPGTT page table level
+ *
+ * This function is used to create a PPGTT mm object from a guest to GVT-g
+ * notification.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu,
+                int page_table_level)
+{
+        u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
+        struct intel_vgpu_mm *mm;
+
+        if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
+                return -EINVAL;
+
+        mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
+        if (mm) {
+                intel_gvt_mm_reference(mm);
+        } else {
+                mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_PPGTT,
+                                pdp, page_table_level, 0);
+                if (IS_ERR(mm)) {
+                        gvt_err("fail to create mm\n");
+                        return PTR_ERR(mm);
+                }
+        }
+        return 0;
+}
+
+/**
+ * intel_vgpu_g2v_destroy_ppgtt_mm - destroy a PPGTT mm object from
+ * g2v notification
+ * @vgpu: a vGPU
+ * @page_table_level: PPGTT page table level
+ *
+ * This function is used to create a PPGTT mm object from a guest to GVT-g
+ * notification.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu,
+                int page_table_level)
+{
+        u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
+        struct intel_vgpu_mm *mm;
+
+        if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
+                return -EINVAL;
+
+        mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
+        if (!mm) {
+                gvt_err("fail to find ppgtt instance.\n");
+                return -EINVAL;
+        }
+        intel_gvt_mm_unreference(mm);
+        return 0;
+}
+
+/**
+ * intel_gvt_init_gtt - initialize mm components of a GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the initialization stage, to initialize
+ * the mm components of a GVT device.
+ *
+ * Returns:
+ * zero on success, negative error code if failed.
+ */
+int intel_gvt_init_gtt(struct intel_gvt *gvt)
+{
+        int ret;
+
+        gvt_dbg_core("init gtt\n");
+
+        if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+                gvt->gtt.pte_ops = &gen8_gtt_pte_ops;
+                gvt->gtt.gma_ops = &gen8_gtt_gma_ops;
+                gvt->gtt.mm_alloc_page_table = gen8_mm_alloc_page_table;
+                gvt->gtt.mm_free_page_table = gen8_mm_free_page_table;
+        } else {
+                return -ENODEV;
+        }
+
+        if (enable_out_of_sync) {
+                ret = setup_spt_oos(gvt);
+                if (ret) {
+                        gvt_err("fail to initialize SPT oos\n");
+                        return ret;
+                }
+        }
+        INIT_LIST_HEAD(&gvt->gtt.mm_lru_list_head);
+        return 0;
+}
+
+/**
+ * intel_gvt_clean_gtt - clean up mm components of a GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the driver unloading stage, to clean up the
+ * the mm components of a GVT device.
+ *
+ */
+void intel_gvt_clean_gtt(struct intel_gvt *gvt)
+{
+        if (enable_out_of_sync)
+                clean_spt_oos(gvt);
+}
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
new file mode 100644
index 000000000000..e4dcde78f3f9
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -0,0 +1,270 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Xiao Zheng <xiao.zheng@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#ifndef _GVT_GTT_H_
+#define _GVT_GTT_H_
+
+#define GTT_PAGE_SHIFT          12
+#define GTT_PAGE_SIZE           (1UL << GTT_PAGE_SHIFT)
+#define GTT_PAGE_MASK           (~(GTT_PAGE_SIZE-1))
+
+struct intel_vgpu_mm;
+
+#define INTEL_GVT_GTT_HASH_BITS 8
+#define INTEL_GVT_INVALID_ADDR (~0UL)
+
+struct intel_gvt_gtt_entry {
+        u64 val64;
+        int type;
+};
+
+struct intel_gvt_gtt_pte_ops {
+        struct intel_gvt_gtt_entry *(*get_entry)(void *pt,
+                struct intel_gvt_gtt_entry *e,
+                unsigned long index, bool hypervisor_access, unsigned long gpa,
+                struct intel_vgpu *vgpu);
+        struct intel_gvt_gtt_entry *(*set_entry)(void *pt,
+                struct intel_gvt_gtt_entry *e,
+                unsigned long index, bool hypervisor_access, unsigned long gpa,
+                struct intel_vgpu *vgpu);
+        bool (*test_present)(struct intel_gvt_gtt_entry *e);
+        void (*clear_present)(struct intel_gvt_gtt_entry *e);
+        bool (*test_pse)(struct intel_gvt_gtt_entry *e);
+        void (*set_pfn)(struct intel_gvt_gtt_entry *e, unsigned long pfn);
+        unsigned long (*get_pfn)(struct intel_gvt_gtt_entry *e);
+};
+
+struct intel_gvt_gtt_gma_ops {
+        unsigned long (*gma_to_ggtt_pte_index)(unsigned long gma);
+        unsigned long (*gma_to_pte_index)(unsigned long gma);
+        unsigned long (*gma_to_pde_index)(unsigned long gma);
+        unsigned long (*gma_to_l3_pdp_index)(unsigned long gma);
+        unsigned long (*gma_to_l4_pdp_index)(unsigned long gma);
+        unsigned long (*gma_to_pml4_index)(unsigned long gma);
+};
+
+struct intel_gvt_gtt {
+        struct intel_gvt_gtt_pte_ops *pte_ops;
+        struct intel_gvt_gtt_gma_ops *gma_ops;
+        int (*mm_alloc_page_table)(struct intel_vgpu_mm *mm);
+        void (*mm_free_page_table)(struct intel_vgpu_mm *mm);
+        struct list_head oos_page_use_list_head;
+        struct list_head oos_page_free_list_head;
+        struct list_head mm_lru_list_head;
+};
+
+enum {
+        INTEL_GVT_MM_GGTT = 0,
+        INTEL_GVT_MM_PPGTT,
+};
+
+struct intel_vgpu_mm {
+        int type;
+        bool initialized;
+        bool shadowed;
+
+        int page_table_entry_type;
+        u32 page_table_entry_size;
+        u32 page_table_entry_cnt;
+        void *virtual_page_table;
+        void *shadow_page_table;
+
+        int page_table_level;
+        bool has_shadow_page_table;
+        u32 pde_base_index;
+
+        struct list_head list;
+        struct kref ref;
+        atomic_t pincount;
+        struct list_head lru_list;
+        struct intel_vgpu *vgpu;
+};
+
+extern struct intel_gvt_gtt_entry *intel_vgpu_mm_get_entry(
+                struct intel_vgpu_mm *mm,
+                void *page_table, struct intel_gvt_gtt_entry *e,
+                unsigned long index);
+
+extern struct intel_gvt_gtt_entry *intel_vgpu_mm_set_entry(
+                struct intel_vgpu_mm *mm,
+                void *page_table, struct intel_gvt_gtt_entry *e,
+                unsigned long index);
+
+#define ggtt_get_guest_entry(mm, e, index) \
+        intel_vgpu_mm_get_entry(mm, mm->virtual_page_table, e, index)
+
+#define ggtt_set_guest_entry(mm, e, index) \
+        intel_vgpu_mm_set_entry(mm, mm->virtual_page_table, e, index)
+
+#define ggtt_get_shadow_entry(mm, e, index) \
+        intel_vgpu_mm_get_entry(mm, mm->shadow_page_table, e, index)
+
+#define ggtt_set_shadow_entry(mm, e, index) \
+        intel_vgpu_mm_set_entry(mm, mm->shadow_page_table, e, index)
+
+#define ppgtt_get_guest_root_entry(mm, e, index) \
+        intel_vgpu_mm_get_entry(mm, mm->virtual_page_table, e, index)
+
+#define ppgtt_set_guest_root_entry(mm, e, index) \
+        intel_vgpu_mm_set_entry(mm, mm->virtual_page_table, e, index)
+
+#define ppgtt_get_shadow_root_entry(mm, e, index) \
+        intel_vgpu_mm_get_entry(mm, mm->shadow_page_table, e, index)
+
+#define ppgtt_set_shadow_root_entry(mm, e, index) \
+        intel_vgpu_mm_set_entry(mm, mm->shadow_page_table, e, index)
+
+extern struct intel_vgpu_mm *intel_vgpu_create_mm(struct intel_vgpu *vgpu,
+                int mm_type, void *virtual_page_table, int page_table_level,
+                u32 pde_base_index);
+extern void intel_vgpu_destroy_mm(struct kref *mm_ref);
+
+struct intel_vgpu_guest_page;
+
+struct intel_vgpu_gtt {
+        struct intel_vgpu_mm *ggtt_mm;
+        unsigned long active_ppgtt_mm_bitmap;
+        struct list_head mm_list_head;
+        DECLARE_HASHTABLE(shadow_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
+        DECLARE_HASHTABLE(guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
+        atomic_t n_write_protected_guest_page;
+        struct list_head oos_page_list_head;
+        struct list_head post_shadow_list_head;
+        struct page *scratch_page;
+        unsigned long scratch_page_mfn;
+};
+
+extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu);
+extern void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu);
+
+extern int intel_gvt_init_gtt(struct intel_gvt *gvt);
+extern void intel_gvt_clean_gtt(struct intel_gvt *gvt);
+
+extern struct intel_vgpu_mm *intel_gvt_find_ppgtt_mm(struct intel_vgpu *vgpu,
+                int page_table_level, void *root_entry);
+
+struct intel_vgpu_oos_page;
+
+struct intel_vgpu_shadow_page {
+        void *vaddr;
+        struct page *page;
+        int type;
+        struct hlist_node node;
+        unsigned long mfn;
+};
+
+struct intel_vgpu_guest_page {
+        struct hlist_node node;
+        bool writeprotection;
+        unsigned long gfn;
+        int (*handler)(void *, u64, void *, int);
+        void *data;
+        unsigned long write_cnt;
+        struct intel_vgpu_oos_page *oos_page;
+};
+
+struct intel_vgpu_oos_page {
+        struct intel_vgpu_guest_page *guest_page;
+        struct list_head list;
+        struct list_head vm_list;
+        int id;
+        unsigned char mem[GTT_PAGE_SIZE];
+};
+
+#define GTT_ENTRY_NUM_IN_ONE_PAGE 512
+
+struct intel_vgpu_ppgtt_spt {
+        struct intel_vgpu_shadow_page shadow_page;
+        struct intel_vgpu_guest_page guest_page;
+        int guest_page_type;
+        atomic_t refcount;
+        struct intel_vgpu *vgpu;
+        DECLARE_BITMAP(post_shadow_bitmap, GTT_ENTRY_NUM_IN_ONE_PAGE);
+        struct list_head post_shadow_list;
+};
+
+int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *guest_page,
+                unsigned long gfn,
+                int (*handler)(void *gp, u64, void *, int),
+                void *data);
+
+void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *guest_page);
+
+int intel_vgpu_set_guest_page_writeprotection(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *guest_page);
+
+void intel_vgpu_clear_guest_page_writeprotection(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *guest_page);
+
+struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+                struct intel_vgpu *vgpu, unsigned long gfn);
+
+int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu);
+
+int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu);
+
+static inline void intel_gvt_mm_reference(struct intel_vgpu_mm *mm)
+{
+        kref_get(&mm->ref);
+}
+
+static inline void intel_gvt_mm_unreference(struct intel_vgpu_mm *mm)
+{
+        kref_put(&mm->ref, intel_vgpu_destroy_mm);
+}
+
+int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm);
+
+void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm);
+
+unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm,
+                unsigned long gma);
+
+struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
+                int page_table_level, void *root_entry);
+
+int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu,
+                int page_table_level);
+
+int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu,
+                int page_table_level);
+
+int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu,
+        unsigned int off, void *p_data, unsigned int bytes);
+
+int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu,
+        unsigned int off, void *p_data, unsigned int bytes);
+
+#endif /* _GVT_GTT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index 927f4579f5b6..31b59d40f3fb 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -19,12 +19,23 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Eddie Dong <eddie.dong@intel.com>
+ *
+ * Contributors:
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #include <linux/types.h>
 #include <xen/xen.h>
+#include <linux/kthread.h>
 
 #include "i915_drv.h"
+#include "gvt.h"
 
 struct intel_gvt_host intel_gvt_host;
 
@@ -33,6 +44,13 @@ static const char * const supported_hypervisors[] = {
         [INTEL_GVT_HYPERVISOR_KVM] = "KVM",
 };
 
+struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops = {
+        .emulate_cfg_read = intel_vgpu_emulate_cfg_read,
+        .emulate_cfg_write = intel_vgpu_emulate_cfg_write,
+        .emulate_mmio_read = intel_vgpu_emulate_mmio_read,
+        .emulate_mmio_write = intel_vgpu_emulate_mmio_write,
+};
+
 /**
  * intel_gvt_init_host - Load MPT modules and detect if we're running in host
  * @gvt: intel gvt device
@@ -84,9 +102,66 @@ int intel_gvt_init_host(void)
 
 static void init_device_info(struct intel_gvt *gvt)
 {
-        if (IS_BROADWELL(gvt->dev_priv))
-                gvt->device_info.max_support_vgpus = 8;
-        /* This function will grow large in GVT device model patches. */
+        struct intel_gvt_device_info *info = &gvt->device_info;
+
+        if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+                info->max_support_vgpus = 8;
+                info->cfg_space_size = 256;
+                info->mmio_size = 2 * 1024 * 1024;
+                info->mmio_bar = 0;
+                info->msi_cap_offset = IS_SKYLAKE(gvt->dev_priv) ? 0xac : 0x90;
+                info->gtt_start_offset = 8 * 1024 * 1024;
+                info->gtt_entry_size = 8;
+                info->gtt_entry_size_shift = 3;
+                info->gmadr_bytes_in_cmd = 8;
+                info->max_surface_size = 36 * 1024 * 1024;
+        }
+}
+
+static int gvt_service_thread(void *data)
+{
+        struct intel_gvt *gvt = (struct intel_gvt *)data;
+        int ret;
+
+        gvt_dbg_core("service thread start\n");
+
+        while (!kthread_should_stop()) {
+                ret = wait_event_interruptible(gvt->service_thread_wq,
+                                kthread_should_stop() || gvt->service_request);
+
+                if (kthread_should_stop())
+                        break;
+
+                if (WARN_ONCE(ret, "service thread is waken up by signal.\n"))
+                        continue;
+
+                if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK,
+                                        (void *)&gvt->service_request)) {
+                        mutex_lock(&gvt->lock);
+                        intel_gvt_emulate_vblank(gvt);
+                        mutex_unlock(&gvt->lock);
+                }
+        }
+
+        return 0;
+}
+
+static void clean_service_thread(struct intel_gvt *gvt)
+{
+        kthread_stop(gvt->service_thread);
+}
+
+static int init_service_thread(struct intel_gvt *gvt)
+{
+        init_waitqueue_head(&gvt->service_thread_wq);
+
+        gvt->service_thread = kthread_run(gvt_service_thread,
+                        gvt, "gvt_service_thread");
+        if (IS_ERR(gvt->service_thread)) {
+                gvt_err("fail to start service thread.\n");
+                return PTR_ERR(gvt->service_thread);
+        }
+        return 0;
 }
 
 /**
@@ -99,14 +174,23 @@ static void init_device_info(struct intel_gvt *gvt)
  */
 void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
 {
-        struct intel_gvt *gvt = &dev_priv->gvt;
+        struct intel_gvt *gvt = to_gvt(dev_priv);
 
-        if (WARN_ON(!gvt->initialized))
+        if (WARN_ON(!gvt))
                 return;
 
-        /* Other de-initialization of GVT components will be introduced. */
+        clean_service_thread(gvt);
+        intel_gvt_clean_cmd_parser(gvt);
+        intel_gvt_clean_sched_policy(gvt);
+        intel_gvt_clean_workload_scheduler(gvt);
+        intel_gvt_clean_opregion(gvt);
+        intel_gvt_clean_gtt(gvt);
+        intel_gvt_clean_irq(gvt);
+        intel_gvt_clean_mmio_info(gvt);
+        intel_gvt_free_firmware(gvt);
 
-        gvt->initialized = false;
+        kfree(dev_priv->gvt);
+        dev_priv->gvt = NULL;
 }
 
 /**
@@ -122,7 +206,9 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
  */
 int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 {
-        struct intel_gvt *gvt = &dev_priv->gvt;
+        struct intel_gvt *gvt;
+        int ret;
+
         /*
          * Cannot initialize GVT device without intel_gvt_host gets
          * initialized first.
@@ -130,16 +216,76 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
         if (WARN_ON(!intel_gvt_host.initialized))
                 return -EINVAL;
 
-        if (WARN_ON(gvt->initialized))
+        if (WARN_ON(dev_priv->gvt))
                 return -EEXIST;
 
+        gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL);
+        if (!gvt)
+                return -ENOMEM;
+
         gvt_dbg_core("init gvt device\n");
 
+        mutex_init(&gvt->lock);
+        gvt->dev_priv = dev_priv;
+
         init_device_info(gvt);
-        /*
-         * Other initialization of GVT components will be introduce here.
-         */
+
+        ret = intel_gvt_setup_mmio_info(gvt);
+        if (ret)
+                return ret;
+
+        ret = intel_gvt_load_firmware(gvt);
+        if (ret)
+                goto out_clean_mmio_info;
+
+        ret = intel_gvt_init_irq(gvt);
+        if (ret)
+                goto out_free_firmware;
+
+        ret = intel_gvt_init_gtt(gvt);
+        if (ret)
+                goto out_clean_irq;
+
+        ret = intel_gvt_init_opregion(gvt);
+        if (ret)
+                goto out_clean_gtt;
+
+        ret = intel_gvt_init_workload_scheduler(gvt);
+        if (ret)
+                goto out_clean_opregion;
+
+        ret = intel_gvt_init_sched_policy(gvt);
+        if (ret)
+                goto out_clean_workload_scheduler;
+
+        ret = intel_gvt_init_cmd_parser(gvt);
+        if (ret)
+                goto out_clean_sched_policy;
+
+        ret = init_service_thread(gvt);
+        if (ret)
+                goto out_clean_cmd_parser;
+
         gvt_dbg_core("gvt device creation is done\n");
-        gvt->initialized = true;
+        dev_priv->gvt = gvt;
         return 0;
+
+out_clean_cmd_parser:
+        intel_gvt_clean_cmd_parser(gvt);
+out_clean_sched_policy:
+        intel_gvt_clean_sched_policy(gvt);
+out_clean_workload_scheduler:
+        intel_gvt_clean_workload_scheduler(gvt);
+out_clean_opregion:
+        intel_gvt_clean_opregion(gvt);
+out_clean_gtt:
+        intel_gvt_clean_gtt(gvt);
+out_clean_irq:
+        intel_gvt_clean_irq(gvt);
+out_free_firmware:
+        intel_gvt_free_firmware(gvt);
+out_clean_mmio_info:
+        intel_gvt_clean_mmio_info(gvt);
+        kfree(gvt);
+        return ret;
 }
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index fb619a6e519d..11df62b542b1 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -19,6 +19,15 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Eddie Dong <eddie.dong@intel.com>
+ *
+ * Contributors:
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_H_
@@ -26,6 +35,17 @@
 
 #include "debug.h"
 #include "hypercall.h"
+#include "mmio.h"
+#include "reg.h"
+#include "interrupt.h"
+#include "gtt.h"
+#include "display.h"
+#include "edid.h"
+#include "execlist.h"
+#include "scheduler.h"
+#include "sched_policy.h"
+#include "render.h"
+#include "cmd_parser.h"
 
 #define GVT_MAX_VGPU 8
 
@@ -45,25 +65,324 @@ extern struct intel_gvt_host intel_gvt_host;
 /* Describe per-platform limitations. */
 struct intel_gvt_device_info {
         u32 max_support_vgpus;
-        /* This data structure will grow bigger in GVT device model patches */
+        u32 cfg_space_size;
+        u32 mmio_size;
+        u32 mmio_bar;
+        unsigned long msi_cap_offset;
+        u32 gtt_start_offset;
+        u32 gtt_entry_size;
+        u32 gtt_entry_size_shift;
+        int gmadr_bytes_in_cmd;
+        u32 max_surface_size;
+};
+
+/* GM resources owned by a vGPU */
+struct intel_vgpu_gm {
+        u64 aperture_sz;
+        u64 hidden_sz;
+        struct drm_mm_node low_gm_node;
+        struct drm_mm_node high_gm_node;
+};
+
+#define INTEL_GVT_MAX_NUM_FENCES 32
+
+/* Fences owned by a vGPU */
+struct intel_vgpu_fence {
+        struct drm_i915_fence_reg *regs[INTEL_GVT_MAX_NUM_FENCES];
+        u32 base;
+        u32 size;
+};
+
+struct intel_vgpu_mmio {
+        void *vreg;
+        void *sreg;
+        bool disable_warn_untrack;
+};
+
+#define INTEL_GVT_MAX_CFG_SPACE_SZ 256
+#define INTEL_GVT_MAX_BAR_NUM 4
+
+struct intel_vgpu_pci_bar {
+        u64 size;
+        bool tracked;
+};
+
+struct intel_vgpu_cfg_space {
+        unsigned char virtual_cfg_space[INTEL_GVT_MAX_CFG_SPACE_SZ];
+        struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM];
+};
+
+#define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space)
+
+#define INTEL_GVT_MAX_PIPE 4
+
+struct intel_vgpu_irq {
+        bool irq_warn_once[INTEL_GVT_EVENT_MAX];
+        DECLARE_BITMAP(flip_done_event[INTEL_GVT_MAX_PIPE],
+                       INTEL_GVT_EVENT_MAX);
+};
+
+struct intel_vgpu_opregion {
+        void *va;
+        u32 gfn[INTEL_GVT_OPREGION_PAGES];
+        struct page *pages[INTEL_GVT_OPREGION_PAGES];
+};
+
+#define vgpu_opregion(vgpu) (&(vgpu->opregion))
+
+#define INTEL_GVT_MAX_PORT 5
+
+struct intel_vgpu_display {
+        struct intel_vgpu_i2c_edid i2c_edid;
+        struct intel_vgpu_port ports[INTEL_GVT_MAX_PORT];
+        struct intel_vgpu_sbi sbi;
 };
 
 struct intel_vgpu {
         struct intel_gvt *gvt;
         int id;
         unsigned long handle; /* vGPU handle used by hypervisor MPT modules */
+        bool active;
+        bool resetting;
+        void *sched_data;
+
+        struct intel_vgpu_fence fence;
+        struct intel_vgpu_gm gm;
+        struct intel_vgpu_cfg_space cfg_space;
+        struct intel_vgpu_mmio mmio;
+        struct intel_vgpu_irq irq;
+        struct intel_vgpu_gtt gtt;
+        struct intel_vgpu_opregion opregion;
+        struct intel_vgpu_display display;
+        struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
+        struct list_head workload_q_head[I915_NUM_ENGINES];
+        struct kmem_cache *workloads;
+        atomic_t running_workload_num;
+        DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
+        struct i915_gem_context *shadow_ctx;
+        struct notifier_block shadow_ctx_notifier_block;
+};
+
+struct intel_gvt_gm {
+        unsigned long vgpu_allocated_low_gm_size;
+        unsigned long vgpu_allocated_high_gm_size;
+};
+
+struct intel_gvt_fence {
+        unsigned long vgpu_allocated_fence_num;
+};
+
+#define INTEL_GVT_MMIO_HASH_BITS 9
+
+struct intel_gvt_mmio {
+        u32 *mmio_attribute;
+        DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS);
+};
+
+struct intel_gvt_firmware {
+        void *cfg_space;
+        void *mmio;
+        bool firmware_loaded;
+};
+
+struct intel_gvt_opregion {
+        void __iomem *opregion_va;
+        u32 opregion_pa;
 };
 
 struct intel_gvt {
         struct mutex lock;
-        bool initialized;
-
         struct drm_i915_private *dev_priv;
         struct idr vgpu_idr;    /* vGPU IDR pool */
 
         struct intel_gvt_device_info device_info;
+        struct intel_gvt_gm gm;
+        struct intel_gvt_fence fence;
+        struct intel_gvt_mmio mmio;
+        struct intel_gvt_firmware firmware;
+        struct intel_gvt_irq irq;
+        struct intel_gvt_gtt gtt;
+        struct intel_gvt_opregion opregion;
+        struct intel_gvt_workload_scheduler scheduler;
+        DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
+
+        struct task_struct *service_thread;
+        wait_queue_head_t service_thread_wq;
+        unsigned long service_request;
 };
 
+static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915)
+{
+        return i915->gvt;
+}
+
+enum {
+        INTEL_GVT_REQUEST_EMULATE_VBLANK = 0,
+};
+
+static inline void intel_gvt_request_service(struct intel_gvt *gvt,
+                int service)
+{
+        set_bit(service, (void *)&gvt->service_request);
+        wake_up(&gvt->service_thread_wq);
+}
+
+void intel_gvt_free_firmware(struct intel_gvt *gvt);
+int intel_gvt_load_firmware(struct intel_gvt *gvt);
+
+/* Aperture/GM space definitions for GVT device */
+#define gvt_aperture_sz(gvt)      (gvt->dev_priv->ggtt.mappable_end)
+#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.mappable_base)
+
+#define gvt_ggtt_gm_sz(gvt)       (gvt->dev_priv->ggtt.base.total)
+#define gvt_ggtt_sz(gvt) \
+        ((gvt->dev_priv->ggtt.base.total >> PAGE_SHIFT) << 3)
+#define gvt_hidden_sz(gvt)        (gvt_ggtt_gm_sz(gvt) - gvt_aperture_sz(gvt))
+
+#define gvt_aperture_gmadr_base(gvt) (0)
+#define gvt_aperture_gmadr_end(gvt) (gvt_aperture_gmadr_base(gvt) \
+                                     + gvt_aperture_sz(gvt) - 1)
+
+#define gvt_hidden_gmadr_base(gvt) (gvt_aperture_gmadr_base(gvt) \
+                                    + gvt_aperture_sz(gvt))
+#define gvt_hidden_gmadr_end(gvt) (gvt_hidden_gmadr_base(gvt) \
+                                   + gvt_hidden_sz(gvt) - 1)
+
+#define gvt_fence_sz(gvt) (gvt->dev_priv->num_fence_regs)
+
+/* Aperture/GM space definitions for vGPU */
+#define vgpu_aperture_offset(vgpu)      ((vgpu)->gm.low_gm_node.start)
+#define vgpu_hidden_offset(vgpu)        ((vgpu)->gm.high_gm_node.start)
+#define vgpu_aperture_sz(vgpu)          ((vgpu)->gm.aperture_sz)
+#define vgpu_hidden_sz(vgpu)            ((vgpu)->gm.hidden_sz)
+
+#define vgpu_aperture_pa_base(vgpu) \
+        (gvt_aperture_pa_base(vgpu->gvt) + vgpu_aperture_offset(vgpu))
+
+#define vgpu_ggtt_gm_sz(vgpu) ((vgpu)->gm.aperture_sz + (vgpu)->gm.hidden_sz)
+
+#define vgpu_aperture_pa_end(vgpu) \
+        (vgpu_aperture_pa_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
+
+#define vgpu_aperture_gmadr_base(vgpu) (vgpu_aperture_offset(vgpu))
+#define vgpu_aperture_gmadr_end(vgpu) \
+        (vgpu_aperture_gmadr_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
+
+#define vgpu_hidden_gmadr_base(vgpu) (vgpu_hidden_offset(vgpu))
+#define vgpu_hidden_gmadr_end(vgpu) \
+        (vgpu_hidden_gmadr_base(vgpu) + vgpu_hidden_sz(vgpu) - 1)
+
+#define vgpu_fence_base(vgpu) (vgpu->fence.base)
+#define vgpu_fence_sz(vgpu) (vgpu->fence.size)
+
+struct intel_vgpu_creation_params {
+        __u64 handle;
+        __u64 low_gm_sz;  /* in MB */
+        __u64 high_gm_sz; /* in MB */
+        __u64 fence_sz;
+        __s32 primary;
+        __u64 vgpu_id;
+};
+
+int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
+                              struct intel_vgpu_creation_params *param);
+void intel_vgpu_free_resource(struct intel_vgpu *vgpu);
+void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
+        u32 fence, u64 value);
+
+/* Macros for easily accessing vGPU virtual/shadow register */
+#define vgpu_vreg(vgpu, reg) \
+        (*(u32 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg8(vgpu, reg) \
+        (*(u8 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg16(vgpu, reg) \
+        (*(u16 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg64(vgpu, reg) \
+        (*(u64 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg(vgpu, reg) \
+        (*(u32 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg8(vgpu, reg) \
+        (*(u8 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg16(vgpu, reg) \
+        (*(u16 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg64(vgpu, reg) \
+        (*(u64 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+
+#define for_each_active_vgpu(gvt, vgpu, id) \
+        idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \
+                for_each_if(vgpu->active)
+
+static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu,
+                                            u32 offset, u32 val, bool low)
+{
+        u32 *pval;
+
+        /* BAR offset should be 32 bits algiend */
+        offset = rounddown(offset, 4);
+        pval = (u32 *)(vgpu_cfg_space(vgpu) + offset);
+
+        if (low) {
+                /*
+                 * only update bit 31 - bit 4,
+                 * leave the bit 3 - bit 0 unchanged.
+                 */
+                *pval = (val & GENMASK(31, 4)) | (*pval & GENMASK(3, 0));
+        }
+}
+
+struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
+                                         struct intel_vgpu_creation_params *
+                                         param);
+
+void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
+
+/* validating GM functions */
+#define vgpu_gmadr_is_aperture(vgpu, gmadr) \
+        ((gmadr >= vgpu_aperture_gmadr_base(vgpu)) && \
+         (gmadr <= vgpu_aperture_gmadr_end(vgpu)))
+
+#define vgpu_gmadr_is_hidden(vgpu, gmadr) \
+        ((gmadr >= vgpu_hidden_gmadr_base(vgpu)) && \
+         (gmadr <= vgpu_hidden_gmadr_end(vgpu)))
+
+#define vgpu_gmadr_is_valid(vgpu, gmadr) \
+         ((vgpu_gmadr_is_aperture(vgpu, gmadr) || \
+          (vgpu_gmadr_is_hidden(vgpu, gmadr))))
+
+#define gvt_gmadr_is_aperture(gvt, gmadr) \
+         ((gmadr >= gvt_aperture_gmadr_base(gvt)) && \
+          (gmadr <= gvt_aperture_gmadr_end(gvt)))
+
+#define gvt_gmadr_is_hidden(gvt, gmadr) \
+          ((gmadr >= gvt_hidden_gmadr_base(gvt)) && \
+           (gmadr <= gvt_hidden_gmadr_end(gvt)))
+
+#define gvt_gmadr_is_valid(gvt, gmadr) \
+          (gvt_gmadr_is_aperture(gvt, gmadr) || \
+            gvt_gmadr_is_hidden(gvt, gmadr))
+
+bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size);
+int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr);
+int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr);
+int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
+                             unsigned long *h_index);
+int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
+                             unsigned long *g_index);
+
+int intel_vgpu_emulate_cfg_read(void *__vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes);
+
+int intel_vgpu_emulate_cfg_write(void *__vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes);
+
+void intel_gvt_clean_opregion(struct intel_gvt *gvt);
+int intel_gvt_init_opregion(struct intel_gvt *gvt);
+
+void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu);
+int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa);
+
+int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
+
 #include "mpt.h"
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
new file mode 100644
index 000000000000..3e74fb3d4aa9
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -0,0 +1,2797 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Pei Zhang <pei.zhang@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+
+/* XXX FIXME i915 has changed PP_XXX definition */
+#define PCH_PP_STATUS  _MMIO(0xc7200)
+#define PCH_PP_CONTROL _MMIO(0xc7204)
+#define PCH_PP_ON_DELAYS _MMIO(0xc7208)
+#define PCH_PP_OFF_DELAYS _MMIO(0xc720c)
+#define PCH_PP_DIVISOR _MMIO(0xc7210)
+
+/* Register contains RO bits */
+#define F_RO            (1 << 0)
+/* Register contains graphics address */
+#define F_GMADR         (1 << 1)
+/* Mode mask registers with high 16 bits as the mask bits */
+#define F_MODE_MASK     (1 << 2)
+/* This reg can be accessed by GPU commands */
+#define F_CMD_ACCESS    (1 << 3)
+/* This reg has been accessed by a VM */
+#define F_ACCESSED      (1 << 4)
+/* This reg has been accessed through GPU commands */
+#define F_CMD_ACCESSED  (1 << 5)
+/* This reg could be accessed by unaligned address */
+#define F_UNALIGN       (1 << 6)
+
+unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt)
+{
+        if (IS_BROADWELL(gvt->dev_priv))
+                return D_BDW;
+        else if (IS_SKYLAKE(gvt->dev_priv))
+                return D_SKL;
+
+        return 0;
+}
+
+bool intel_gvt_match_device(struct intel_gvt *gvt,
+                unsigned long device)
+{
+        return intel_gvt_get_device_type(gvt) & device;
+}
+
+static void read_vreg(struct intel_vgpu *vgpu, unsigned int offset,
+        void *p_data, unsigned int bytes)
+{
+        memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+}
+
+static void write_vreg(struct intel_vgpu *vgpu, unsigned int offset,
+        void *p_data, unsigned int bytes)
+{
+        memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes);
+}
+
+static int new_mmio_info(struct intel_gvt *gvt,
+                u32 offset, u32 flags, u32 size,
+                u32 addr_mask, u32 ro_mask, u32 device,
+                void *read, void *write)
+{
+        struct intel_gvt_mmio_info *info, *p;
+        u32 start, end, i;
+
+        if (!intel_gvt_match_device(gvt, device))
+                return 0;
+
+        if (WARN_ON(!IS_ALIGNED(offset, 4)))
+                return -EINVAL;
+
+        start = offset;
+        end = offset + size;
+
+        for (i = start; i < end; i += 4) {
+                info = kzalloc(sizeof(*info), GFP_KERNEL);
+                if (!info)
+                        return -ENOMEM;
+
+                info->offset = i;
+                p = intel_gvt_find_mmio_info(gvt, info->offset);
+                if (p)
+                        gvt_err("dup mmio definition offset %x\n",
+                                info->offset);
+                info->size = size;
+                info->length = (i + 4) < end ? 4 : (end - i);
+                info->addr_mask = addr_mask;
+                info->device = device;
+                info->read = read ? read : intel_vgpu_default_mmio_read;
+                info->write = write ? write : intel_vgpu_default_mmio_write;
+                gvt->mmio.mmio_attribute[info->offset / 4] = flags;
+                INIT_HLIST_NODE(&info->node);
+                hash_add(gvt->mmio.mmio_info_table, &info->node, info->offset);
+        }
+        return 0;
+}
+
+static int render_mmio_to_ring_id(struct intel_gvt *gvt, unsigned int reg)
+{
+        enum intel_engine_id id;
+        struct intel_engine_cs *engine;
+
+        reg &= ~GENMASK(11, 0);
+        for_each_engine(engine, gvt->dev_priv, id) {
+                if (engine->mmio_base == reg)
+                        return id;
+        }
+        return -1;
+}
+
+#define offset_to_fence_num(offset) \
+        ((offset - i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0))) >> 3)
+
+#define fence_num_to_offset(num) \
+        (num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0)))
+
+static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
+                unsigned int fence_num, void *p_data, unsigned int bytes)
+{
+        if (fence_num >= vgpu_fence_sz(vgpu)) {
+                gvt_err("vgpu%d: found oob fence register access\n",
+                                vgpu->id);
+                gvt_err("vgpu%d: total fence num %d access fence num %d\n",
+                                vgpu->id, vgpu_fence_sz(vgpu), fence_num);
+                memset(p_data, 0, bytes);
+        }
+        return 0;
+}
+
+static int fence_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
+                void *p_data, unsigned int bytes)
+{
+        int ret;
+
+        ret = sanitize_fence_mmio_access(vgpu, offset_to_fence_num(off),
+                        p_data, bytes);
+        if (ret)
+                return ret;
+        read_vreg(vgpu, off, p_data, bytes);
+        return 0;
+}
+
+static int fence_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
+                void *p_data, unsigned int bytes)
+{
+        unsigned int fence_num = offset_to_fence_num(off);
+        int ret;
+
+        ret = sanitize_fence_mmio_access(vgpu, fence_num, p_data, bytes);
+        if (ret)
+                return ret;
+        write_vreg(vgpu, off, p_data, bytes);
+
+        intel_vgpu_write_fence(vgpu, fence_num,
+                        vgpu_vreg64(vgpu, fence_num_to_offset(fence_num)));
+        return 0;
+}
+
+#define CALC_MODE_MASK_REG(old, new) \
+        (((new) & GENMASK(31, 16)) \
+         | ((((old) & GENMASK(15, 0)) & ~((new) >> 16)) \
+         | ((new) & ((new) >> 16))))
+
+static int mul_force_wake_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u32 old, new;
+        uint32_t ack_reg_offset;
+
+        old = vgpu_vreg(vgpu, offset);
+        new = CALC_MODE_MASK_REG(old, *(u32 *)p_data);
+
+        if (IS_SKYLAKE(vgpu->gvt->dev_priv)) {
+                switch (offset) {
+                case FORCEWAKE_RENDER_GEN9_REG:
+                        ack_reg_offset = FORCEWAKE_ACK_RENDER_GEN9_REG;
+                        break;
+                case FORCEWAKE_BLITTER_GEN9_REG:
+                        ack_reg_offset = FORCEWAKE_ACK_BLITTER_GEN9_REG;
+                        break;
+                case FORCEWAKE_MEDIA_GEN9_REG:
+                        ack_reg_offset = FORCEWAKE_ACK_MEDIA_GEN9_REG;
+                        break;
+                default:
+                        /*should not hit here*/
+                        gvt_err("invalid forcewake offset 0x%x\n", offset);
+                        return 1;
+                }
+        } else {
+                ack_reg_offset = FORCEWAKE_ACK_HSW_REG;
+        }
+
+        vgpu_vreg(vgpu, offset) = new;
+        vgpu_vreg(vgpu, ack_reg_offset) = (new & GENMASK(15, 0));
+        return 0;
+}
+
+static int handle_device_reset(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes, unsigned long bitmap)
+{
+        struct intel_gvt_workload_scheduler *scheduler =
+                &vgpu->gvt->scheduler;
+
+        vgpu->resetting = true;
+
+        intel_vgpu_stop_schedule(vgpu);
+        if (scheduler->current_vgpu == vgpu) {
+                mutex_unlock(&vgpu->gvt->lock);
+                intel_gvt_wait_vgpu_idle(vgpu);
+                mutex_lock(&vgpu->gvt->lock);
+        }
+
+        intel_vgpu_reset_execlist(vgpu, bitmap);
+
+        vgpu->resetting = false;
+
+        return 0;
+}
+
+static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 data;
+        u64 bitmap = 0;
+
+        data = vgpu_vreg(vgpu, offset);
+
+        if (data & GEN6_GRDOM_FULL) {
+                gvt_dbg_mmio("vgpu%d: request full GPU reset\n", vgpu->id);
+                bitmap = 0xff;
+        }
+        if (data & GEN6_GRDOM_RENDER) {
+                gvt_dbg_mmio("vgpu%d: request RCS reset\n", vgpu->id);
+                bitmap |= (1 << RCS);
+        }
+        if (data & GEN6_GRDOM_MEDIA) {
+                gvt_dbg_mmio("vgpu%d: request VCS reset\n", vgpu->id);
+                bitmap |= (1 << VCS);
+        }
+        if (data & GEN6_GRDOM_BLT) {
+                gvt_dbg_mmio("vgpu%d: request BCS Reset\n", vgpu->id);
+                bitmap |= (1 << BCS);
+        }
+        if (data & GEN6_GRDOM_VECS) {
+                gvt_dbg_mmio("vgpu%d: request VECS Reset\n", vgpu->id);
+                bitmap |= (1 << VECS);
+        }
+        if (data & GEN8_GRDOM_MEDIA2) {
+                gvt_dbg_mmio("vgpu%d: request VCS2 Reset\n", vgpu->id);
+                if (HAS_BSD2(vgpu->gvt->dev_priv))
+                        bitmap |= (1 << VCS2);
+        }
+        return handle_device_reset(vgpu, offset, p_data, bytes, bitmap);
+}
+
+static int gmbus_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        return intel_gvt_i2c_handle_gmbus_read(vgpu, offset, p_data, bytes);
+}
+
+static int gmbus_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        return intel_gvt_i2c_handle_gmbus_write(vgpu, offset, p_data, bytes);
+}
+
+static int pch_pp_control_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        if (vgpu_vreg(vgpu, offset) & PANEL_POWER_ON) {
+                vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_ON;
+                vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE;
+                vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN;
+                vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE;
+
+        } else
+                vgpu_vreg(vgpu, PCH_PP_STATUS) &=
+                        ~(PP_ON | PP_SEQUENCE_POWER_DOWN
+                                        | PP_CYCLE_DELAY_ACTIVE);
+        return 0;
+}
+
+static int transconf_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        if (vgpu_vreg(vgpu, offset) & TRANS_ENABLE)
+                vgpu_vreg(vgpu, offset) |= TRANS_STATE_ENABLE;
+        else
+                vgpu_vreg(vgpu, offset) &= ~TRANS_STATE_ENABLE;
+        return 0;
+}
+
+static int lcpll_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        if (vgpu_vreg(vgpu, offset) & LCPLL_PLL_DISABLE)
+                vgpu_vreg(vgpu, offset) &= ~LCPLL_PLL_LOCK;
+        else
+                vgpu_vreg(vgpu, offset) |= LCPLL_PLL_LOCK;
+
+        if (vgpu_vreg(vgpu, offset) & LCPLL_CD_SOURCE_FCLK)
+                vgpu_vreg(vgpu, offset) |= LCPLL_CD_SOURCE_FCLK_DONE;
+        else
+                vgpu_vreg(vgpu, offset) &= ~LCPLL_CD_SOURCE_FCLK_DONE;
+
+        return 0;
+}
+
+static int dpy_reg_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        *(u32 *)p_data = (1 << 17);
+        return 0;
+}
+
+static int dpy_reg_mmio_read_2(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        *(u32 *)p_data = 3;
+        return 0;
+}
+
+static int dpy_reg_mmio_read_3(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        *(u32 *)p_data = (0x2f << 16);
+        return 0;
+}
+
+static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 data;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        data = vgpu_vreg(vgpu, offset);
+
+        if (data & PIPECONF_ENABLE)
+                vgpu_vreg(vgpu, offset) |= I965_PIPECONF_ACTIVE;
+        else
+                vgpu_vreg(vgpu, offset) &= ~I965_PIPECONF_ACTIVE;
+        intel_gvt_check_vblank_emulation(vgpu->gvt);
+        return 0;
+}
+
+static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        if (vgpu_vreg(vgpu, offset) & DDI_BUF_CTL_ENABLE) {
+                vgpu_vreg(vgpu, offset) &= ~DDI_BUF_IS_IDLE;
+        } else {
+                vgpu_vreg(vgpu, offset) |= DDI_BUF_IS_IDLE;
+                if (offset == i915_mmio_reg_offset(DDI_BUF_CTL(PORT_E)))
+                        vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E))
+                                &= ~DP_TP_STATUS_AUTOTRAIN_DONE;
+        }
+        return 0;
+}
+
+static int fdi_rx_iir_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        vgpu_vreg(vgpu, offset) &= ~*(u32 *)p_data;
+        return 0;
+}
+
+#define FDI_LINK_TRAIN_PATTERN1         0
+#define FDI_LINK_TRAIN_PATTERN2         1
+
+static int fdi_auto_training_started(struct intel_vgpu *vgpu)
+{
+        u32 ddi_buf_ctl = vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_E));
+        u32 rx_ctl = vgpu_vreg(vgpu, _FDI_RXA_CTL);
+        u32 tx_ctl = vgpu_vreg(vgpu, DP_TP_CTL(PORT_E));
+
+        if ((ddi_buf_ctl & DDI_BUF_CTL_ENABLE) &&
+                        (rx_ctl & FDI_RX_ENABLE) &&
+                        (rx_ctl & FDI_AUTO_TRAINING) &&
+                        (tx_ctl & DP_TP_CTL_ENABLE) &&
+                        (tx_ctl & DP_TP_CTL_FDI_AUTOTRAIN))
+                return 1;
+        else
+                return 0;
+}
+
+static int check_fdi_rx_train_status(struct intel_vgpu *vgpu,
+                enum pipe pipe, unsigned int train_pattern)
+{
+        i915_reg_t fdi_rx_imr, fdi_tx_ctl, fdi_rx_ctl;
+        unsigned int fdi_rx_check_bits, fdi_tx_check_bits;
+        unsigned int fdi_rx_train_bits, fdi_tx_train_bits;
+        unsigned int fdi_iir_check_bits;
+
+        fdi_rx_imr = FDI_RX_IMR(pipe);
+        fdi_tx_ctl = FDI_TX_CTL(pipe);
+        fdi_rx_ctl = FDI_RX_CTL(pipe);
+
+        if (train_pattern == FDI_LINK_TRAIN_PATTERN1) {
+                fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_1_CPT;
+                fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_1;
+                fdi_iir_check_bits = FDI_RX_BIT_LOCK;
+        } else if (train_pattern == FDI_LINK_TRAIN_PATTERN2) {
+                fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_2_CPT;
+                fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_2;
+                fdi_iir_check_bits = FDI_RX_SYMBOL_LOCK;
+        } else {
+                gvt_err("Invalid train pattern %d\n", train_pattern);
+                return -EINVAL;
+        }
+
+        fdi_rx_check_bits = FDI_RX_ENABLE | fdi_rx_train_bits;
+        fdi_tx_check_bits = FDI_TX_ENABLE | fdi_tx_train_bits;
+
+        /* If imr bit has been masked */
+        if (vgpu_vreg(vgpu, fdi_rx_imr) & fdi_iir_check_bits)
+                return 0;
+
+        if (((vgpu_vreg(vgpu, fdi_tx_ctl) & fdi_tx_check_bits)
+                        == fdi_tx_check_bits)
+                && ((vgpu_vreg(vgpu, fdi_rx_ctl) & fdi_rx_check_bits)
+                        == fdi_rx_check_bits))
+                return 1;
+        else
+                return 0;
+}
+
+#define INVALID_INDEX (~0U)
+
+static unsigned int calc_index(unsigned int offset, unsigned int start,
+        unsigned int next, unsigned int end, i915_reg_t i915_end)
+{
+        unsigned int range = next - start;
+
+        if (!end)
+                end = i915_mmio_reg_offset(i915_end);
+        if (offset < start || offset > end)
+                return INVALID_INDEX;
+        offset -= start;
+        return offset / range;
+}
+
+#define FDI_RX_CTL_TO_PIPE(offset) \
+        calc_index(offset, _FDI_RXA_CTL, _FDI_RXB_CTL, 0, FDI_RX_CTL(PIPE_C))
+
+#define FDI_TX_CTL_TO_PIPE(offset) \
+        calc_index(offset, _FDI_TXA_CTL, _FDI_TXB_CTL, 0, FDI_TX_CTL(PIPE_C))
+
+#define FDI_RX_IMR_TO_PIPE(offset) \
+        calc_index(offset, _FDI_RXA_IMR, _FDI_RXB_IMR, 0, FDI_RX_IMR(PIPE_C))
+
+static int update_fdi_rx_iir_status(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        i915_reg_t fdi_rx_iir;
+        unsigned int index;
+        int ret;
+
+        if (FDI_RX_CTL_TO_PIPE(offset) != INVALID_INDEX)
+                index = FDI_RX_CTL_TO_PIPE(offset);
+        else if (FDI_TX_CTL_TO_PIPE(offset) != INVALID_INDEX)
+                index = FDI_TX_CTL_TO_PIPE(offset);
+        else if (FDI_RX_IMR_TO_PIPE(offset) != INVALID_INDEX)
+                index = FDI_RX_IMR_TO_PIPE(offset);
+        else {
+                gvt_err("Unsupport registers %x\n", offset);
+                return -EINVAL;
+        }
+
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        fdi_rx_iir = FDI_RX_IIR(index);
+
+        ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN1);
+        if (ret < 0)
+                return ret;
+        if (ret)
+                vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK;
+
+        ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN2);
+        if (ret < 0)
+                return ret;
+        if (ret)
+                vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK;
+
+        if (offset == _FDI_RXA_CTL)
+                if (fdi_auto_training_started(vgpu))
+                        vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E)) |=
+                                DP_TP_STATUS_AUTOTRAIN_DONE;
+        return 0;
+}
+
+#define DP_TP_CTL_TO_PORT(offset) \
+        calc_index(offset, _DP_TP_CTL_A, _DP_TP_CTL_B, 0, DP_TP_CTL(PORT_E))
+
+static int dp_tp_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        i915_reg_t status_reg;
+        unsigned int index;
+        u32 data;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        index = DP_TP_CTL_TO_PORT(offset);
+        data = (vgpu_vreg(vgpu, offset) & GENMASK(10, 8)) >> 8;
+        if (data == 0x2) {
+                status_reg = DP_TP_STATUS(index);
+                vgpu_vreg(vgpu, status_reg) |= (1 << 25);
+        }
+        return 0;
+}
+
+static int dp_tp_status_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u32 reg_val;
+        u32 sticky_mask;
+
+        reg_val = *((u32 *)p_data);
+        sticky_mask = GENMASK(27, 26) | (1 << 24);
+
+        vgpu_vreg(vgpu, offset) = (reg_val & ~sticky_mask) |
+                (vgpu_vreg(vgpu, offset) & sticky_mask);
+        vgpu_vreg(vgpu, offset) &= ~(reg_val & sticky_mask);
+        return 0;
+}
+
+static int pch_adpa_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u32 data;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        data = vgpu_vreg(vgpu, offset);
+
+        if (data & ADPA_CRT_HOTPLUG_FORCE_TRIGGER)
+                vgpu_vreg(vgpu, offset) &= ~ADPA_CRT_HOTPLUG_FORCE_TRIGGER;
+        return 0;
+}
+
+static int south_chicken2_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u32 data;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        data = vgpu_vreg(vgpu, offset);
+
+        if (data & FDI_MPHY_IOSFSB_RESET_CTL)
+                vgpu_vreg(vgpu, offset) |= FDI_MPHY_IOSFSB_RESET_STATUS;
+        else
+                vgpu_vreg(vgpu, offset) &= ~FDI_MPHY_IOSFSB_RESET_STATUS;
+        return 0;
+}
+
+#define DSPSURF_TO_PIPE(offset) \
+        calc_index(offset, _DSPASURF, _DSPBSURF, 0, DSPSURF(PIPE_C))
+
+static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        unsigned int index = DSPSURF_TO_PIPE(offset);
+        i915_reg_t surflive_reg = DSPSURFLIVE(index);
+        int flip_event[] = {
+                [PIPE_A] = PRIMARY_A_FLIP_DONE,
+                [PIPE_B] = PRIMARY_B_FLIP_DONE,
+                [PIPE_C] = PRIMARY_C_FLIP_DONE,
+        };
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+
+        set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
+        return 0;
+}
+
+#define SPRSURF_TO_PIPE(offset) \
+        calc_index(offset, _SPRA_SURF, _SPRB_SURF, 0, SPRSURF(PIPE_C))
+
+static int spr_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        unsigned int index = SPRSURF_TO_PIPE(offset);
+        i915_reg_t surflive_reg = SPRSURFLIVE(index);
+        int flip_event[] = {
+                [PIPE_A] = SPRITE_A_FLIP_DONE,
+                [PIPE_B] = SPRITE_B_FLIP_DONE,
+                [PIPE_C] = SPRITE_C_FLIP_DONE,
+        };
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+
+        set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
+        return 0;
+}
+
+static int trigger_aux_channel_interrupt(struct intel_vgpu *vgpu,
+                unsigned int reg)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        enum intel_gvt_event_type event;
+
+        if (reg == _DPA_AUX_CH_CTL)
+                event = AUX_CHANNEL_A;
+        else if (reg == _PCH_DPB_AUX_CH_CTL || reg == _DPB_AUX_CH_CTL)
+                event = AUX_CHANNEL_B;
+        else if (reg == _PCH_DPC_AUX_CH_CTL || reg == _DPC_AUX_CH_CTL)
+                event = AUX_CHANNEL_C;
+        else if (reg == _PCH_DPD_AUX_CH_CTL || reg == _DPD_AUX_CH_CTL)
+                event = AUX_CHANNEL_D;
+        else {
+                WARN_ON(true);
+                return -EINVAL;
+        }
+
+        intel_vgpu_trigger_virtual_event(vgpu, event);
+        return 0;
+}
+
+static int dp_aux_ch_ctl_trans_done(struct intel_vgpu *vgpu, u32 value,
+                unsigned int reg, int len, bool data_valid)
+{
+        /* mark transaction done */
+        value |= DP_AUX_CH_CTL_DONE;
+        value &= ~DP_AUX_CH_CTL_SEND_BUSY;
+        value &= ~DP_AUX_CH_CTL_RECEIVE_ERROR;
+
+        if (data_valid)
+                value &= ~DP_AUX_CH_CTL_TIME_OUT_ERROR;
+        else
+                value |= DP_AUX_CH_CTL_TIME_OUT_ERROR;
+
+        /* message size */
+        value &= ~(0xf << 20);
+        value |= (len << 20);
+        vgpu_vreg(vgpu, reg) = value;
+
+        if (value & DP_AUX_CH_CTL_INTERRUPT)
+                return trigger_aux_channel_interrupt(vgpu, reg);
+        return 0;
+}
+
+static void dp_aux_ch_ctl_link_training(struct intel_vgpu_dpcd_data *dpcd,
+                uint8_t t)
+{
+        if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == DPCD_TRAINING_PATTERN_1) {
+                /* training pattern 1 for CR */
+                /* set LANE0_CR_DONE, LANE1_CR_DONE */
+                dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_CR_DONE;
+                /* set LANE2_CR_DONE, LANE3_CR_DONE */
+                dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_CR_DONE;
+        } else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) ==
+                        DPCD_TRAINING_PATTERN_2) {
+                /* training pattern 2 for EQ */
+                /* Set CHANNEL_EQ_DONE and  SYMBOL_LOCKED for Lane0_1 */
+                dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_EQ_DONE;
+                dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_SYMBOL_LOCKED;
+                /* Set CHANNEL_EQ_DONE and  SYMBOL_LOCKED for Lane2_3 */
+                dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_EQ_DONE;
+                dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_SYMBOL_LOCKED;
+                /* set INTERLANE_ALIGN_DONE */
+                dpcd->data[DPCD_LANE_ALIGN_STATUS_UPDATED] |=
+                        DPCD_INTERLANE_ALIGN_DONE;
+        } else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) ==
+                        DPCD_LINK_TRAINING_DISABLED) {
+                /* finish link training */
+                /* set sink status as synchronized */
+                dpcd->data[DPCD_SINK_STATUS] = DPCD_SINK_IN_SYNC;
+        }
+}
+
+#define _REG_HSW_DP_AUX_CH_CTL(dp) \
+        ((dp) ? (_PCH_DPB_AUX_CH_CTL + ((dp)-1)*0x100) : 0x64010)
+
+#define _REG_SKL_DP_AUX_CH_CTL(dp) (0x64010 + (dp) * 0x100)
+
+#define OFFSET_TO_DP_AUX_PORT(offset) (((offset) & 0xF00) >> 8)
+
+#define dpy_is_valid_port(port) \
+                (((port) >= PORT_A) && ((port) < I915_MAX_PORTS))
+
+static int dp_aux_ch_ctl_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        struct intel_vgpu_display *display = &vgpu->display;
+        int msg, addr, ctrl, op, len;
+        int port_index = OFFSET_TO_DP_AUX_PORT(offset);
+        struct intel_vgpu_dpcd_data *dpcd = NULL;
+        struct intel_vgpu_port *port = NULL;
+        u32 data;
+
+        if (!dpy_is_valid_port(port_index)) {
+                gvt_err("GVT(%d): Unsupported DP port access!\n", vgpu->id);
+                return 0;
+        }
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        data = vgpu_vreg(vgpu, offset);
+
+        if (IS_SKYLAKE(vgpu->gvt->dev_priv) &&
+            offset != _REG_SKL_DP_AUX_CH_CTL(port_index)) {
+                /* SKL DPB/C/D aux ctl register changed */
+                return 0;
+        } else if (IS_BROADWELL(vgpu->gvt->dev_priv) &&
+                   offset != _REG_HSW_DP_AUX_CH_CTL(port_index)) {
+                /* write to the data registers */
+                return 0;
+        }
+
+        if (!(data & DP_AUX_CH_CTL_SEND_BUSY)) {
+                /* just want to clear the sticky bits */
+                vgpu_vreg(vgpu, offset) = 0;
+                return 0;
+        }
+
+        port = &display->ports[port_index];
+        dpcd = port->dpcd;
+
+        /* read out message from DATA1 register */
+        msg = vgpu_vreg(vgpu, offset + 4);
+        addr = (msg >> 8) & 0xffff;
+        ctrl = (msg >> 24) & 0xff;
+        len = msg & 0xff;
+        op = ctrl >> 4;
+
+        if (op == GVT_AUX_NATIVE_WRITE) {
+                int t;
+                uint8_t buf[16];
+
+                if ((addr + len + 1) >= DPCD_SIZE) {
+                        /*
+                         * Write request exceeds what we supported,
+                         * DCPD spec: When a Source Device is writing a DPCD
+                         * address not supported by the Sink Device, the Sink
+                         * Device shall reply with AUX NACK and “M” equal to
+                         * zero.
+                         */
+
+                        /* NAK the write */
+                        vgpu_vreg(vgpu, offset + 4) = AUX_NATIVE_REPLY_NAK;
+                        dp_aux_ch_ctl_trans_done(vgpu, data, offset, 2, true);
+                        return 0;
+                }
+
+                /*
+                 * Write request format: (command + address) occupies
+                 * 3 bytes, followed by (len + 1) bytes of data.
+                 */
+                if (WARN_ON((len + 4) > AUX_BURST_SIZE))
+                        return -EINVAL;
+
+                /* unpack data from vreg to buf */
+                for (t = 0; t < 4; t++) {
+                        u32 r = vgpu_vreg(vgpu, offset + 8 + t * 4);
+
+                        buf[t * 4] = (r >> 24) & 0xff;
+                        buf[t * 4 + 1] = (r >> 16) & 0xff;
+                        buf[t * 4 + 2] = (r >> 8) & 0xff;
+                        buf[t * 4 + 3] = r & 0xff;
+                }
+
+                /* write to virtual DPCD */
+                if (dpcd && dpcd->data_valid) {
+                        for (t = 0; t <= len; t++) {
+                                int p = addr + t;
+
+                                dpcd->data[p] = buf[t];
+                                /* check for link training */
+                                if (p == DPCD_TRAINING_PATTERN_SET)
+                                        dp_aux_ch_ctl_link_training(dpcd,
+                                                        buf[t]);
+                        }
+                }
+
+                /* ACK the write */
+                vgpu_vreg(vgpu, offset + 4) = 0;
+                dp_aux_ch_ctl_trans_done(vgpu, data, offset, 1,
+                                dpcd && dpcd->data_valid);
+                return 0;
+        }
+
+        if (op == GVT_AUX_NATIVE_READ) {
+                int idx, i, ret = 0;
+
+                if ((addr + len + 1) >= DPCD_SIZE) {
+                        /*
+                         * read request exceeds what we supported
+                         * DPCD spec: A Sink Device receiving a Native AUX CH
+                         * read request for an unsupported DPCD address must
+                         * reply with an AUX ACK and read data set equal to
+                         * zero instead of replying with AUX NACK.
+                         */
+
+                        /* ACK the READ*/
+                        vgpu_vreg(vgpu, offset + 4) = 0;
+                        vgpu_vreg(vgpu, offset + 8) = 0;
+                        vgpu_vreg(vgpu, offset + 12) = 0;
+                        vgpu_vreg(vgpu, offset + 16) = 0;
+                        vgpu_vreg(vgpu, offset + 20) = 0;
+
+                        dp_aux_ch_ctl_trans_done(vgpu, data, offset, len + 2,
+                                        true);
+                        return 0;
+                }
+
+                for (idx = 1; idx <= 5; idx++) {
+                        /* clear the data registers */
+                        vgpu_vreg(vgpu, offset + 4 * idx) = 0;
+                }
+
+                /*
+                 * Read reply format: ACK (1 byte) plus (len + 1) bytes of data.
+                 */
+                if (WARN_ON((len + 2) > AUX_BURST_SIZE))
+                        return -EINVAL;
+
+                /* read from virtual DPCD to vreg */
+                /* first 4 bytes: [ACK][addr][addr+1][addr+2] */
+                if (dpcd && dpcd->data_valid) {
+                        for (i = 1; i <= (len + 1); i++) {
+                                int t;
+
+                                t = dpcd->data[addr + i - 1];
+                                t <<= (24 - 8 * (i % 4));
+                                ret |= t;
+
+                                if ((i % 4 == 3) || (i == (len + 1))) {
+                                        vgpu_vreg(vgpu, offset +
+                                                        (i / 4 + 1) * 4) = ret;
+                                        ret = 0;
+                                }
+                        }
+                }
+                dp_aux_ch_ctl_trans_done(vgpu, data, offset, len + 2,
+                                dpcd && dpcd->data_valid);
+                return 0;
+        }
+
+        /* i2c transaction starts */
+        intel_gvt_i2c_handle_aux_ch_write(vgpu, port_index, offset, p_data);
+
+        if (data & DP_AUX_CH_CTL_INTERRUPT)
+                trigger_aux_channel_interrupt(vgpu, offset);
+        return 0;
+}
+
+static int vga_control_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        bool vga_disable;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        vga_disable = vgpu_vreg(vgpu, offset) & VGA_DISP_DISABLE;
+
+        gvt_dbg_core("vgpu%d: %s VGA mode\n", vgpu->id,
+                        vga_disable ? "Disable" : "Enable");
+        return 0;
+}
+
+static u32 read_virtual_sbi_register(struct intel_vgpu *vgpu,
+                unsigned int sbi_offset)
+{
+        struct intel_vgpu_display *display = &vgpu->display;
+        int num = display->sbi.number;
+        int i;
+
+        for (i = 0; i < num; ++i)
+                if (display->sbi.registers[i].offset == sbi_offset)
+                        break;
+
+        if (i == num)
+                return 0;
+
+        return display->sbi.registers[i].value;
+}
+
+static void write_virtual_sbi_register(struct intel_vgpu *vgpu,
+                unsigned int offset, u32 value)
+{
+        struct intel_vgpu_display *display = &vgpu->display;
+        int num = display->sbi.number;
+        int i;
+
+        for (i = 0; i < num; ++i) {
+                if (display->sbi.registers[i].offset == offset)
+                        break;
+        }
+
+        if (i == num) {
+                if (num == SBI_REG_MAX) {
+                        gvt_err("vgpu%d: SBI caching meets maximum limits\n",
+                                        vgpu->id);
+                        return;
+                }
+                display->sbi.number++;
+        }
+
+        display->sbi.registers[i].offset = offset;
+        display->sbi.registers[i].value = value;
+}
+
+static int sbi_data_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
+                                SBI_OPCODE_SHIFT) == SBI_CMD_CRRD) {
+                unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) &
+                                SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT;
+                vgpu_vreg(vgpu, offset) = read_virtual_sbi_register(vgpu,
+                                sbi_offset);
+        }
+        read_vreg(vgpu, offset, p_data, bytes);
+        return 0;
+}
+
+static bool sbi_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 data;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        data = vgpu_vreg(vgpu, offset);
+
+        data &= ~(SBI_STAT_MASK << SBI_STAT_SHIFT);
+        data |= SBI_READY;
+
+        data &= ~(SBI_RESPONSE_MASK << SBI_RESPONSE_SHIFT);
+        data |= SBI_RESPONSE_SUCCESS;
+
+        vgpu_vreg(vgpu, offset) = data;
+
+        if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
+                                SBI_OPCODE_SHIFT) == SBI_CMD_CRWR) {
+                unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) &
+                                SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT;
+
+                write_virtual_sbi_register(vgpu, sbi_offset,
+                                vgpu_vreg(vgpu, SBI_DATA));
+        }
+        return 0;
+}
+
+#define _vgtif_reg(x) \
+        (VGT_PVINFO_PAGE + offsetof(struct vgt_if, x))
+
+static int pvinfo_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        bool invalid_read = false;
+
+        read_vreg(vgpu, offset, p_data, bytes);
+
+        switch (offset) {
+        case _vgtif_reg(magic) ... _vgtif_reg(vgt_id):
+                if (offset + bytes > _vgtif_reg(vgt_id) + 4)
+                        invalid_read = true;
+                break;
+        case _vgtif_reg(avail_rs.mappable_gmadr.base) ...
+                _vgtif_reg(avail_rs.fence_num):
+                if (offset + bytes >
+                        _vgtif_reg(avail_rs.fence_num) + 4)
+                        invalid_read = true;
+                break;
+        case 0x78010:   /* vgt_caps */
+        case 0x7881c:
+                break;
+        default:
+                invalid_read = true;
+                break;
+        }
+        if (invalid_read)
+                gvt_err("invalid pvinfo read: [%x:%x] = %x\n",
+                                offset, bytes, *(u32 *)p_data);
+        return 0;
+}
+
+static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification)
+{
+        int ret = 0;
+
+        switch (notification) {
+        case VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE:
+                ret = intel_vgpu_g2v_create_ppgtt_mm(vgpu, 3);
+                break;
+        case VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY:
+                ret = intel_vgpu_g2v_destroy_ppgtt_mm(vgpu, 3);
+                break;
+        case VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE:
+                ret = intel_vgpu_g2v_create_ppgtt_mm(vgpu, 4);
+                break;
+        case VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY:
+                ret = intel_vgpu_g2v_destroy_ppgtt_mm(vgpu, 4);
+                break;
+        case VGT_G2V_EXECLIST_CONTEXT_CREATE:
+        case VGT_G2V_EXECLIST_CONTEXT_DESTROY:
+        case 1: /* Remove this in guest driver. */
+                break;
+        default:
+                gvt_err("Invalid PV notification %d\n", notification);
+        }
+        return ret;
+}
+
+static int send_display_ready_uevent(struct intel_vgpu *vgpu, int ready)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
+        char *env[3] = {NULL, NULL, NULL};
+        char vmid_str[20];
+        char display_ready_str[20];
+
+        snprintf(display_ready_str, 20, "GVT_DISPLAY_READY=%d\n", ready);
+        env[0] = display_ready_str;
+
+        snprintf(vmid_str, 20, "VMID=%d", vgpu->id);
+        env[1] = vmid_str;
+
+        return kobject_uevent_env(kobj, KOBJ_ADD, env);
+}
+
+static int pvinfo_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 data;
+        int ret;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        data = vgpu_vreg(vgpu, offset);
+
+        switch (offset) {
+        case _vgtif_reg(display_ready):
+                send_display_ready_uevent(vgpu, data ? 1 : 0);
+                break;
+        case _vgtif_reg(g2v_notify):
+                ret = handle_g2v_notification(vgpu, data);
+                break;
+        /* add xhot and yhot to handled list to avoid error log */
+        case 0x78830:
+        case 0x78834:
+        case _vgtif_reg(pdp[0].lo):
+        case _vgtif_reg(pdp[0].hi):
+        case _vgtif_reg(pdp[1].lo):
+        case _vgtif_reg(pdp[1].hi):
+        case _vgtif_reg(pdp[2].lo):
+        case _vgtif_reg(pdp[2].hi):
+        case _vgtif_reg(pdp[3].lo):
+        case _vgtif_reg(pdp[3].hi):
+        case _vgtif_reg(execlist_context_descriptor_lo):
+        case _vgtif_reg(execlist_context_descriptor_hi):
+                break;
+        default:
+                gvt_err("invalid pvinfo write offset %x bytes %x data %x\n",
+                                offset, bytes, data);
+                break;
+        }
+        return 0;
+}
+
+static int pf_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u32 val = *(u32 *)p_data;
+
+        if ((offset == _PS_1A_CTRL || offset == _PS_2A_CTRL ||
+           offset == _PS_1B_CTRL || offset == _PS_2B_CTRL ||
+           offset == _PS_1C_CTRL) && (val & PS_PLANE_SEL_MASK) != 0) {
+                WARN_ONCE(true, "VM(%d): guest is trying to scaling a plane\n",
+                          vgpu->id);
+                return 0;
+        }
+
+        return intel_vgpu_default_mmio_write(vgpu, offset, p_data, bytes);
+}
+
+static int power_well_ctl_mmio_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        if (vgpu_vreg(vgpu, offset) & HSW_PWR_WELL_ENABLE_REQUEST)
+                vgpu_vreg(vgpu, offset) |= HSW_PWR_WELL_STATE_ENABLED;
+        else
+                vgpu_vreg(vgpu, offset) &= ~HSW_PWR_WELL_STATE_ENABLED;
+        return 0;
+}
+
+static int fpga_dbg_mmio_write(struct intel_vgpu *vgpu,
+        unsigned int offset, void *p_data, unsigned int bytes)
+{
+        write_vreg(vgpu, offset, p_data, bytes);
+
+        if (vgpu_vreg(vgpu, offset) & FPGA_DBG_RM_NOCLAIM)
+                vgpu_vreg(vgpu, offset) &= ~FPGA_DBG_RM_NOCLAIM;
+        return 0;
+}
+
+static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 mode = *(u32 *)p_data;
+
+        if (GFX_MODE_BIT_SET_IN_MASK(mode, START_DMA)) {
+                WARN_ONCE(1, "VM(%d): iGVT-g doesn't supporte GuC\n",
+                                vgpu->id);
+                return 0;
+        }
+
+        return 0;
+}
+
+static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        u32 trtte = *(u32 *)p_data;
+
+        if ((trtte & 1) && (trtte & (1 << 1)) == 0) {
+                WARN(1, "VM(%d): Use physical address for TRTT!\n",
+                                vgpu->id);
+                return -EINVAL;
+        }
+        write_vreg(vgpu, offset, p_data, bytes);
+        /* TRTTE is not per-context */
+        I915_WRITE(_MMIO(offset), vgpu_vreg(vgpu, offset));
+
+        return 0;
+}
+
+static int gen9_trtt_chicken_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        u32 val = *(u32 *)p_data;
+
+        if (val & 1) {
+                /* unblock hw logic */
+                I915_WRITE(_MMIO(offset), val);
+        }
+        write_vreg(vgpu, offset, p_data, bytes);
+        return 0;
+}
+
+static int dpll_status_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 v = 0;
+
+        if (vgpu_vreg(vgpu, 0x46010) & (1 << 31))
+                v |= (1 << 0);
+
+        if (vgpu_vreg(vgpu, 0x46014) & (1 << 31))
+                v |= (1 << 8);
+
+        if (vgpu_vreg(vgpu, 0x46040) & (1 << 31))
+                v |= (1 << 16);
+
+        if (vgpu_vreg(vgpu, 0x46060) & (1 << 31))
+                v |= (1 << 24);
+
+        vgpu_vreg(vgpu, offset) = v;
+
+        return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
+}
+
+static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 value = *(u32 *)p_data;
+        u32 cmd = value & 0xff;
+        u32 *data0 = &vgpu_vreg(vgpu, GEN6_PCODE_DATA);
+
+        switch (cmd) {
+        case 0x6:
+                /**
+                 * "Read memory latency" command on gen9.
+                 * Below memory latency values are read
+                 * from skylake platform.
+                 */
+                if (!*data0)
+                        *data0 = 0x1e1a1100;
+                else
+                        *data0 = 0x61514b3d;
+                break;
+        case 0x5:
+                *data0 |= 0x1;
+                break;
+        }
+
+        gvt_dbg_core("VM(%d) write %x to mailbox, return data0 %x\n",
+                     vgpu->id, value, *data0);
+
+        value &= ~(1 << 31);
+        return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes);
+}
+
+static int skl_power_well_ctl_write(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u32 v = *(u32 *)p_data;
+
+        v &= (1 << 31) | (1 << 29) | (1 << 9) |
+             (1 << 7) | (1 << 5) | (1 << 3) | (1 << 1);
+        v |= (v >> 1);
+
+        return intel_vgpu_default_mmio_write(vgpu, offset, &v, bytes);
+}
+
+static int skl_misc_ctl_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        i915_reg_t reg = {.reg = offset};
+
+        switch (offset) {
+        case 0x4ddc:
+                vgpu_vreg(vgpu, offset) = 0x8000003c;
+                break;
+        case 0x42080:
+                vgpu_vreg(vgpu, offset) = 0x8000;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        /**
+         * TODO: need detect stepping info after gvt contain such information
+         * 0x4ddc enabled after C0, 0x42080 enabled after E0.
+         */
+        I915_WRITE(reg, vgpu_vreg(vgpu, offset));
+        return 0;
+}
+
+static int skl_lcpll_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 v = *(u32 *)p_data;
+
+        /* other bits are MBZ. */
+        v &= (1 << 31) | (1 << 30);
+        v & (1 << 31) ? (v |= (1 << 30)) : (v &= ~(1 << 30));
+
+        vgpu_vreg(vgpu, offset) = v;
+
+        return 0;
+}
+
+static int ring_timestamp_mmio_read(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+        vgpu_vreg(vgpu, offset) = I915_READ(_MMIO(offset));
+        return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
+}
+
+static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset);
+        struct intel_vgpu_execlist *execlist;
+        u32 data = *(u32 *)p_data;
+        int ret;
+
+        if (WARN_ON(ring_id < 0 || ring_id > I915_NUM_ENGINES - 1))
+                return -EINVAL;
+
+        execlist = &vgpu->execlist[ring_id];
+
+        execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data;
+        if (execlist->elsp_dwords.index == 3)
+                ret = intel_vgpu_submit_execlist(vgpu, ring_id);
+
+        ++execlist->elsp_dwords.index;
+        execlist->elsp_dwords.index &= 0x3;
+        return 0;
+}
+
+static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 data = *(u32 *)p_data;
+        int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset);
+        bool enable_execlist;
+
+        write_vreg(vgpu, offset, p_data, bytes);
+        if ((data & _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE))
+                        || (data & _MASKED_BIT_DISABLE(GFX_RUN_LIST_ENABLE))) {
+                enable_execlist = !!(data & GFX_RUN_LIST_ENABLE);
+
+                gvt_dbg_core("EXECLIST %s on ring %d\n",
+                                (enable_execlist ? "enabling" : "disabling"),
+                                ring_id);
+
+                if (enable_execlist)
+                        intel_vgpu_start_schedule(vgpu);
+        }
+        return 0;
+}
+
+static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
+                unsigned int offset, void *p_data, unsigned int bytes)
+{
+        int rc = 0;
+        unsigned int id = 0;
+
+        switch (offset) {
+        case 0x4260:
+                id = RCS;
+                break;
+        case 0x4264:
+                id = VCS;
+                break;
+        case 0x4268:
+                id = VCS2;
+                break;
+        case 0x426c:
+                id = BCS;
+                break;
+        case 0x4270:
+                id = VECS;
+                break;
+        default:
+                rc = -EINVAL;
+                break;
+        }
+        set_bit(id, (void *)vgpu->tlb_handle_pending);
+
+        return rc;
+}
+
+#define MMIO_F(reg, s, f, am, rm, d, r, w) do { \
+        ret = new_mmio_info(gvt, INTEL_GVT_MMIO_OFFSET(reg), \
+                f, s, am, rm, d, r, w); \
+        if (ret) \
+                return ret; \
+} while (0)
+
+#define MMIO_D(reg, d) \
+        MMIO_F(reg, 4, 0, 0, 0, d, NULL, NULL)
+
+#define MMIO_DH(reg, d, r, w) \
+        MMIO_F(reg, 4, 0, 0, 0, d, r, w)
+
+#define MMIO_DFH(reg, d, f, r, w) \
+        MMIO_F(reg, 4, f, 0, 0, d, r, w)
+
+#define MMIO_GM(reg, d, r, w) \
+        MMIO_F(reg, 4, F_GMADR, 0xFFFFF000, 0, d, r, w)
+
+#define MMIO_RO(reg, d, f, rm, r, w) \
+        MMIO_F(reg, 4, F_RO | f, 0, rm, d, r, w)
+
+#define MMIO_RING_F(prefix, s, f, am, rm, d, r, w) do { \
+        MMIO_F(prefix(RENDER_RING_BASE), s, f, am, rm, d, r, w); \
+        MMIO_F(prefix(BLT_RING_BASE), s, f, am, rm, d, r, w); \
+        MMIO_F(prefix(GEN6_BSD_RING_BASE), s, f, am, rm, d, r, w); \
+        MMIO_F(prefix(VEBOX_RING_BASE), s, f, am, rm, d, r, w); \
+} while (0)
+
+#define MMIO_RING_D(prefix, d) \
+        MMIO_RING_F(prefix, 4, 0, 0, 0, d, NULL, NULL)
+
+#define MMIO_RING_DFH(prefix, d, f, r, w) \
+        MMIO_RING_F(prefix, 4, f, 0, 0, d, r, w)
+
+#define MMIO_RING_GM(prefix, d, r, w) \
+        MMIO_RING_F(prefix, 4, F_GMADR, 0xFFFF0000, 0, d, r, w)
+
+#define MMIO_RING_RO(prefix, d, f, rm, r, w) \
+        MMIO_RING_F(prefix, 4, F_RO | f, 0, rm, d, r, w)
+
+static int init_generic_mmio_info(struct intel_gvt *gvt)
+{
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        int ret;
+
+        MMIO_RING_DFH(RING_IMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler);
+
+        MMIO_DFH(SDEIMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DFH(SDEIER, D_ALL, 0, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DFH(SDEIIR, D_ALL, 0, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(SDEISR, D_ALL);
+
+        MMIO_RING_D(RING_HWSTAM, D_ALL);
+
+        MMIO_GM(RENDER_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+        MMIO_GM(BSD_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+        MMIO_GM(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+        MMIO_GM(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+
+#define RING_REG(base) (base + 0x28)
+        MMIO_RING_D(RING_REG, D_ALL);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x134)
+        MMIO_RING_D(RING_REG, D_ALL);
+#undef RING_REG
+
+        MMIO_GM(0x2148, D_ALL, NULL, NULL);
+        MMIO_GM(CCID, D_ALL, NULL, NULL);
+        MMIO_GM(0x12198, D_ALL, NULL, NULL);
+        MMIO_D(GEN7_CXT_SIZE, D_ALL);
+
+        MMIO_RING_D(RING_TAIL, D_ALL);
+        MMIO_RING_D(RING_HEAD, D_ALL);
+        MMIO_RING_D(RING_CTL, D_ALL);
+        MMIO_RING_D(RING_ACTHD, D_ALL);
+        MMIO_RING_GM(RING_START, D_ALL, NULL, NULL);
+
+        /* RING MODE */
+#define RING_REG(base) (base + 0x29c)
+        MMIO_RING_DFH(RING_REG, D_ALL, F_MODE_MASK, NULL, ring_mode_mmio_write);
+#undef RING_REG
+
+        MMIO_RING_DFH(RING_MI_MODE, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_RING_DFH(RING_INSTPM, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_RING_DFH(RING_TIMESTAMP, D_ALL, F_CMD_ACCESS,
+                        ring_timestamp_mmio_read, NULL);
+        MMIO_RING_DFH(RING_TIMESTAMP_UDW, D_ALL, F_CMD_ACCESS,
+                        ring_timestamp_mmio_read, NULL);
+
+        MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK, NULL, NULL);
+
+        MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0x2088, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0x20e4, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0x2470, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_D(GAM_ECOCHK, D_ALL);
+        MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_D(0x9030, D_ALL);
+        MMIO_D(0x20a0, D_ALL);
+        MMIO_D(0x2420, D_ALL);
+        MMIO_D(0x2430, D_ALL);
+        MMIO_D(0x2434, D_ALL);
+        MMIO_D(0x2438, D_ALL);
+        MMIO_D(0x243c, D_ALL);
+        MMIO_DFH(0x7018, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0xe184, D_ALL, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0xe100, D_ALL, F_MODE_MASK, NULL, NULL);
+
+        /* display */
+        MMIO_F(0x60220, 0x20, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_D(0x602a0, D_ALL);
+
+        MMIO_D(0x65050, D_ALL);
+        MMIO_D(0x650b4, D_ALL);
+
+        MMIO_D(0xc4040, D_ALL);
+        MMIO_D(DERRMR, D_ALL);
+
+        MMIO_D(PIPEDSL(PIPE_A), D_ALL);
+        MMIO_D(PIPEDSL(PIPE_B), D_ALL);
+        MMIO_D(PIPEDSL(PIPE_C), D_ALL);
+        MMIO_D(PIPEDSL(_PIPE_EDP), D_ALL);
+
+        MMIO_DH(PIPECONF(PIPE_A), D_ALL, NULL, pipeconf_mmio_write);
+        MMIO_DH(PIPECONF(PIPE_B), D_ALL, NULL, pipeconf_mmio_write);
+        MMIO_DH(PIPECONF(PIPE_C), D_ALL, NULL, pipeconf_mmio_write);
+        MMIO_DH(PIPECONF(_PIPE_EDP), D_ALL, NULL, pipeconf_mmio_write);
+
+        MMIO_D(PIPESTAT(PIPE_A), D_ALL);
+        MMIO_D(PIPESTAT(PIPE_B), D_ALL);
+        MMIO_D(PIPESTAT(PIPE_C), D_ALL);
+        MMIO_D(PIPESTAT(_PIPE_EDP), D_ALL);
+
+        MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_A), D_ALL);
+        MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_B), D_ALL);
+        MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_C), D_ALL);
+        MMIO_D(PIPE_FLIPCOUNT_G4X(_PIPE_EDP), D_ALL);
+
+        MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_A), D_ALL);
+        MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_B), D_ALL);
+        MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_C), D_ALL);
+        MMIO_D(PIPE_FRMCOUNT_G4X(_PIPE_EDP), D_ALL);
+
+        MMIO_D(CURCNTR(PIPE_A), D_ALL);
+        MMIO_D(CURCNTR(PIPE_B), D_ALL);
+        MMIO_D(CURCNTR(PIPE_C), D_ALL);
+
+        MMIO_D(CURPOS(PIPE_A), D_ALL);
+        MMIO_D(CURPOS(PIPE_B), D_ALL);
+        MMIO_D(CURPOS(PIPE_C), D_ALL);
+
+        MMIO_D(CURBASE(PIPE_A), D_ALL);
+        MMIO_D(CURBASE(PIPE_B), D_ALL);
+        MMIO_D(CURBASE(PIPE_C), D_ALL);
+
+        MMIO_D(0x700ac, D_ALL);
+        MMIO_D(0x710ac, D_ALL);
+        MMIO_D(0x720ac, D_ALL);
+
+        MMIO_D(0x70090, D_ALL);
+        MMIO_D(0x70094, D_ALL);
+        MMIO_D(0x70098, D_ALL);
+        MMIO_D(0x7009c, D_ALL);
+
+        MMIO_D(DSPCNTR(PIPE_A), D_ALL);
+        MMIO_D(DSPADDR(PIPE_A), D_ALL);
+        MMIO_D(DSPSTRIDE(PIPE_A), D_ALL);
+        MMIO_D(DSPPOS(PIPE_A), D_ALL);
+        MMIO_D(DSPSIZE(PIPE_A), D_ALL);
+        MMIO_DH(DSPSURF(PIPE_A), D_ALL, NULL, pri_surf_mmio_write);
+        MMIO_D(DSPOFFSET(PIPE_A), D_ALL);
+        MMIO_D(DSPSURFLIVE(PIPE_A), D_ALL);
+
+        MMIO_D(DSPCNTR(PIPE_B), D_ALL);
+        MMIO_D(DSPADDR(PIPE_B), D_ALL);
+        MMIO_D(DSPSTRIDE(PIPE_B), D_ALL);
+        MMIO_D(DSPPOS(PIPE_B), D_ALL);
+        MMIO_D(DSPSIZE(PIPE_B), D_ALL);
+        MMIO_DH(DSPSURF(PIPE_B), D_ALL, NULL, pri_surf_mmio_write);
+        MMIO_D(DSPOFFSET(PIPE_B), D_ALL);
+        MMIO_D(DSPSURFLIVE(PIPE_B), D_ALL);
+
+        MMIO_D(DSPCNTR(PIPE_C), D_ALL);
+        MMIO_D(DSPADDR(PIPE_C), D_ALL);
+        MMIO_D(DSPSTRIDE(PIPE_C), D_ALL);
+        MMIO_D(DSPPOS(PIPE_C), D_ALL);
+        MMIO_D(DSPSIZE(PIPE_C), D_ALL);
+        MMIO_DH(DSPSURF(PIPE_C), D_ALL, NULL, pri_surf_mmio_write);
+        MMIO_D(DSPOFFSET(PIPE_C), D_ALL);
+        MMIO_D(DSPSURFLIVE(PIPE_C), D_ALL);
+
+        MMIO_D(SPRCTL(PIPE_A), D_ALL);
+        MMIO_D(SPRLINOFF(PIPE_A), D_ALL);
+        MMIO_D(SPRSTRIDE(PIPE_A), D_ALL);
+        MMIO_D(SPRPOS(PIPE_A), D_ALL);
+        MMIO_D(SPRSIZE(PIPE_A), D_ALL);
+        MMIO_D(SPRKEYVAL(PIPE_A), D_ALL);
+        MMIO_D(SPRKEYMSK(PIPE_A), D_ALL);
+        MMIO_DH(SPRSURF(PIPE_A), D_ALL, NULL, spr_surf_mmio_write);
+        MMIO_D(SPRKEYMAX(PIPE_A), D_ALL);
+        MMIO_D(SPROFFSET(PIPE_A), D_ALL);
+        MMIO_D(SPRSCALE(PIPE_A), D_ALL);
+        MMIO_D(SPRSURFLIVE(PIPE_A), D_ALL);
+
+        MMIO_D(SPRCTL(PIPE_B), D_ALL);
+        MMIO_D(SPRLINOFF(PIPE_B), D_ALL);
+        MMIO_D(SPRSTRIDE(PIPE_B), D_ALL);
+        MMIO_D(SPRPOS(PIPE_B), D_ALL);
+        MMIO_D(SPRSIZE(PIPE_B), D_ALL);
+        MMIO_D(SPRKEYVAL(PIPE_B), D_ALL);
+        MMIO_D(SPRKEYMSK(PIPE_B), D_ALL);
+        MMIO_DH(SPRSURF(PIPE_B), D_ALL, NULL, spr_surf_mmio_write);
+        MMIO_D(SPRKEYMAX(PIPE_B), D_ALL);
+        MMIO_D(SPROFFSET(PIPE_B), D_ALL);
+        MMIO_D(SPRSCALE(PIPE_B), D_ALL);
+        MMIO_D(SPRSURFLIVE(PIPE_B), D_ALL);
+
+        MMIO_D(SPRCTL(PIPE_C), D_ALL);
+        MMIO_D(SPRLINOFF(PIPE_C), D_ALL);
+        MMIO_D(SPRSTRIDE(PIPE_C), D_ALL);
+        MMIO_D(SPRPOS(PIPE_C), D_ALL);
+        MMIO_D(SPRSIZE(PIPE_C), D_ALL);
+        MMIO_D(SPRKEYVAL(PIPE_C), D_ALL);
+        MMIO_D(SPRKEYMSK(PIPE_C), D_ALL);
+        MMIO_DH(SPRSURF(PIPE_C), D_ALL, NULL, spr_surf_mmio_write);
+        MMIO_D(SPRKEYMAX(PIPE_C), D_ALL);
+        MMIO_D(SPROFFSET(PIPE_C), D_ALL);
+        MMIO_D(SPRSCALE(PIPE_C), D_ALL);
+        MMIO_D(SPRSURFLIVE(PIPE_C), D_ALL);
+
+        MMIO_F(LGC_PALETTE(PIPE_A, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(LGC_PALETTE(PIPE_B, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(LGC_PALETTE(PIPE_C, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(HTOTAL(TRANSCODER_A), D_ALL);
+        MMIO_D(HBLANK(TRANSCODER_A), D_ALL);
+        MMIO_D(HSYNC(TRANSCODER_A), D_ALL);
+        MMIO_D(VTOTAL(TRANSCODER_A), D_ALL);
+        MMIO_D(VBLANK(TRANSCODER_A), D_ALL);
+        MMIO_D(VSYNC(TRANSCODER_A), D_ALL);
+        MMIO_D(BCLRPAT(TRANSCODER_A), D_ALL);
+        MMIO_D(VSYNCSHIFT(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPESRC(TRANSCODER_A), D_ALL);
+
+        MMIO_D(HTOTAL(TRANSCODER_B), D_ALL);
+        MMIO_D(HBLANK(TRANSCODER_B), D_ALL);
+        MMIO_D(HSYNC(TRANSCODER_B), D_ALL);
+        MMIO_D(VTOTAL(TRANSCODER_B), D_ALL);
+        MMIO_D(VBLANK(TRANSCODER_B), D_ALL);
+        MMIO_D(VSYNC(TRANSCODER_B), D_ALL);
+        MMIO_D(BCLRPAT(TRANSCODER_B), D_ALL);
+        MMIO_D(VSYNCSHIFT(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPESRC(TRANSCODER_B), D_ALL);
+
+        MMIO_D(HTOTAL(TRANSCODER_C), D_ALL);
+        MMIO_D(HBLANK(TRANSCODER_C), D_ALL);
+        MMIO_D(HSYNC(TRANSCODER_C), D_ALL);
+        MMIO_D(VTOTAL(TRANSCODER_C), D_ALL);
+        MMIO_D(VBLANK(TRANSCODER_C), D_ALL);
+        MMIO_D(VSYNC(TRANSCODER_C), D_ALL);
+        MMIO_D(BCLRPAT(TRANSCODER_C), D_ALL);
+        MMIO_D(VSYNCSHIFT(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPESRC(TRANSCODER_C), D_ALL);
+
+        MMIO_D(HTOTAL(TRANSCODER_EDP), D_ALL);
+        MMIO_D(HBLANK(TRANSCODER_EDP), D_ALL);
+        MMIO_D(HSYNC(TRANSCODER_EDP), D_ALL);
+        MMIO_D(VTOTAL(TRANSCODER_EDP), D_ALL);
+        MMIO_D(VBLANK(TRANSCODER_EDP), D_ALL);
+        MMIO_D(VSYNC(TRANSCODER_EDP), D_ALL);
+        MMIO_D(BCLRPAT(TRANSCODER_EDP), D_ALL);
+        MMIO_D(VSYNCSHIFT(TRANSCODER_EDP), D_ALL);
+
+        MMIO_D(PIPE_DATA_M1(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPE_DATA_N1(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPE_DATA_M2(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPE_DATA_N2(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPE_LINK_M1(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPE_LINK_N1(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPE_LINK_M2(TRANSCODER_A), D_ALL);
+        MMIO_D(PIPE_LINK_N2(TRANSCODER_A), D_ALL);
+
+        MMIO_D(PIPE_DATA_M1(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPE_DATA_N1(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPE_DATA_M2(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPE_DATA_N2(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPE_LINK_M1(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPE_LINK_N1(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPE_LINK_M2(TRANSCODER_B), D_ALL);
+        MMIO_D(PIPE_LINK_N2(TRANSCODER_B), D_ALL);
+
+        MMIO_D(PIPE_DATA_M1(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPE_DATA_N1(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPE_DATA_M2(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPE_DATA_N2(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPE_LINK_M1(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPE_LINK_N1(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPE_LINK_M2(TRANSCODER_C), D_ALL);
+        MMIO_D(PIPE_LINK_N2(TRANSCODER_C), D_ALL);
+
+        MMIO_D(PIPE_DATA_M1(TRANSCODER_EDP), D_ALL);
+        MMIO_D(PIPE_DATA_N1(TRANSCODER_EDP), D_ALL);
+        MMIO_D(PIPE_DATA_M2(TRANSCODER_EDP), D_ALL);
+        MMIO_D(PIPE_DATA_N2(TRANSCODER_EDP), D_ALL);
+        MMIO_D(PIPE_LINK_M1(TRANSCODER_EDP), D_ALL);
+        MMIO_D(PIPE_LINK_N1(TRANSCODER_EDP), D_ALL);
+        MMIO_D(PIPE_LINK_M2(TRANSCODER_EDP), D_ALL);
+        MMIO_D(PIPE_LINK_N2(TRANSCODER_EDP), D_ALL);
+
+        MMIO_D(PF_CTL(PIPE_A), D_ALL);
+        MMIO_D(PF_WIN_SZ(PIPE_A), D_ALL);
+        MMIO_D(PF_WIN_POS(PIPE_A), D_ALL);
+        MMIO_D(PF_VSCALE(PIPE_A), D_ALL);
+        MMIO_D(PF_HSCALE(PIPE_A), D_ALL);
+
+        MMIO_D(PF_CTL(PIPE_B), D_ALL);
+        MMIO_D(PF_WIN_SZ(PIPE_B), D_ALL);
+        MMIO_D(PF_WIN_POS(PIPE_B), D_ALL);
+        MMIO_D(PF_VSCALE(PIPE_B), D_ALL);
+        MMIO_D(PF_HSCALE(PIPE_B), D_ALL);
+
+        MMIO_D(PF_CTL(PIPE_C), D_ALL);
+        MMIO_D(PF_WIN_SZ(PIPE_C), D_ALL);
+        MMIO_D(PF_WIN_POS(PIPE_C), D_ALL);
+        MMIO_D(PF_VSCALE(PIPE_C), D_ALL);
+        MMIO_D(PF_HSCALE(PIPE_C), D_ALL);
+
+        MMIO_D(WM0_PIPEA_ILK, D_ALL);
+        MMIO_D(WM0_PIPEB_ILK, D_ALL);
+        MMIO_D(WM0_PIPEC_IVB, D_ALL);
+        MMIO_D(WM1_LP_ILK, D_ALL);
+        MMIO_D(WM2_LP_ILK, D_ALL);
+        MMIO_D(WM3_LP_ILK, D_ALL);
+        MMIO_D(WM1S_LP_ILK, D_ALL);
+        MMIO_D(WM2S_LP_IVB, D_ALL);
+        MMIO_D(WM3S_LP_IVB, D_ALL);
+
+        MMIO_D(BLC_PWM_CPU_CTL2, D_ALL);
+        MMIO_D(BLC_PWM_CPU_CTL, D_ALL);
+        MMIO_D(BLC_PWM_PCH_CTL1, D_ALL);
+        MMIO_D(BLC_PWM_PCH_CTL2, D_ALL);
+
+        MMIO_D(0x48268, D_ALL);
+
+        MMIO_F(PCH_GMBUS0, 4 * 4, 0, 0, 0, D_ALL, gmbus_mmio_read,
+                gmbus_mmio_write);
+        MMIO_F(PCH_GPIOA, 6 * 4, F_UNALIGN, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0xe4f00, 0x28, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_F(_PCH_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+                dp_aux_ch_ctl_mmio_write);
+        MMIO_F(_PCH_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+                dp_aux_ch_ctl_mmio_write);
+        MMIO_F(_PCH_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+                dp_aux_ch_ctl_mmio_write);
+
+        MMIO_RO(PCH_ADPA, D_ALL, 0, ADPA_CRT_HOTPLUG_MONITOR_MASK, NULL, pch_adpa_mmio_write);
+
+        MMIO_DH(_PCH_TRANSACONF, D_ALL, NULL, transconf_mmio_write);
+        MMIO_DH(_PCH_TRANSBCONF, D_ALL, NULL, transconf_mmio_write);
+
+        MMIO_DH(FDI_RX_IIR(PIPE_A), D_ALL, NULL, fdi_rx_iir_mmio_write);
+        MMIO_DH(FDI_RX_IIR(PIPE_B), D_ALL, NULL, fdi_rx_iir_mmio_write);
+        MMIO_DH(FDI_RX_IIR(PIPE_C), D_ALL, NULL, fdi_rx_iir_mmio_write);
+        MMIO_DH(FDI_RX_IMR(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status);
+        MMIO_DH(FDI_RX_IMR(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status);
+        MMIO_DH(FDI_RX_IMR(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status);
+        MMIO_DH(FDI_RX_CTL(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status);
+        MMIO_DH(FDI_RX_CTL(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status);
+        MMIO_DH(FDI_RX_CTL(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status);
+
+        MMIO_D(_PCH_TRANS_HTOTAL_A, D_ALL);
+        MMIO_D(_PCH_TRANS_HBLANK_A, D_ALL);
+        MMIO_D(_PCH_TRANS_HSYNC_A, D_ALL);
+        MMIO_D(_PCH_TRANS_VTOTAL_A, D_ALL);
+        MMIO_D(_PCH_TRANS_VBLANK_A, D_ALL);
+        MMIO_D(_PCH_TRANS_VSYNC_A, D_ALL);
+        MMIO_D(_PCH_TRANS_VSYNCSHIFT_A, D_ALL);
+
+        MMIO_D(_PCH_TRANS_HTOTAL_B, D_ALL);
+        MMIO_D(_PCH_TRANS_HBLANK_B, D_ALL);
+        MMIO_D(_PCH_TRANS_HSYNC_B, D_ALL);
+        MMIO_D(_PCH_TRANS_VTOTAL_B, D_ALL);
+        MMIO_D(_PCH_TRANS_VBLANK_B, D_ALL);
+        MMIO_D(_PCH_TRANS_VSYNC_B, D_ALL);
+        MMIO_D(_PCH_TRANS_VSYNCSHIFT_B, D_ALL);
+
+        MMIO_D(_PCH_TRANSA_DATA_M1, D_ALL);
+        MMIO_D(_PCH_TRANSA_DATA_N1, D_ALL);
+        MMIO_D(_PCH_TRANSA_DATA_M2, D_ALL);
+        MMIO_D(_PCH_TRANSA_DATA_N2, D_ALL);
+        MMIO_D(_PCH_TRANSA_LINK_M1, D_ALL);
+        MMIO_D(_PCH_TRANSA_LINK_N1, D_ALL);
+        MMIO_D(_PCH_TRANSA_LINK_M2, D_ALL);
+        MMIO_D(_PCH_TRANSA_LINK_N2, D_ALL);
+
+        MMIO_D(TRANS_DP_CTL(PIPE_A), D_ALL);
+        MMIO_D(TRANS_DP_CTL(PIPE_B), D_ALL);
+        MMIO_D(TRANS_DP_CTL(PIPE_C), D_ALL);
+
+        MMIO_D(TVIDEO_DIP_CTL(PIPE_A), D_ALL);
+        MMIO_D(TVIDEO_DIP_DATA(PIPE_A), D_ALL);
+        MMIO_D(TVIDEO_DIP_GCP(PIPE_A), D_ALL);
+
+        MMIO_D(TVIDEO_DIP_CTL(PIPE_B), D_ALL);
+        MMIO_D(TVIDEO_DIP_DATA(PIPE_B), D_ALL);
+        MMIO_D(TVIDEO_DIP_GCP(PIPE_B), D_ALL);
+
+        MMIO_D(TVIDEO_DIP_CTL(PIPE_C), D_ALL);
+        MMIO_D(TVIDEO_DIP_DATA(PIPE_C), D_ALL);
+        MMIO_D(TVIDEO_DIP_GCP(PIPE_C), D_ALL);
+
+        MMIO_D(_FDI_RXA_MISC, D_ALL);
+        MMIO_D(_FDI_RXB_MISC, D_ALL);
+        MMIO_D(_FDI_RXA_TUSIZE1, D_ALL);
+        MMIO_D(_FDI_RXA_TUSIZE2, D_ALL);
+        MMIO_D(_FDI_RXB_TUSIZE1, D_ALL);
+        MMIO_D(_FDI_RXB_TUSIZE2, D_ALL);
+
+        MMIO_DH(PCH_PP_CONTROL, D_ALL, NULL, pch_pp_control_mmio_write);
+        MMIO_D(PCH_PP_DIVISOR, D_ALL);
+        MMIO_D(PCH_PP_STATUS,  D_ALL);
+        MMIO_D(PCH_LVDS, D_ALL);
+        MMIO_D(_PCH_DPLL_A, D_ALL);
+        MMIO_D(_PCH_DPLL_B, D_ALL);
+        MMIO_D(_PCH_FPA0, D_ALL);
+        MMIO_D(_PCH_FPA1, D_ALL);
+        MMIO_D(_PCH_FPB0, D_ALL);
+        MMIO_D(_PCH_FPB1, D_ALL);
+        MMIO_D(PCH_DREF_CONTROL, D_ALL);
+        MMIO_D(PCH_RAWCLK_FREQ, D_ALL);
+        MMIO_D(PCH_DPLL_SEL, D_ALL);
+
+        MMIO_D(0x61208, D_ALL);
+        MMIO_D(0x6120c, D_ALL);
+        MMIO_D(PCH_PP_ON_DELAYS, D_ALL);
+        MMIO_D(PCH_PP_OFF_DELAYS, D_ALL);
+
+        MMIO_DH(0xe651c, D_ALL, dpy_reg_mmio_read, NULL);
+        MMIO_DH(0xe661c, D_ALL, dpy_reg_mmio_read, NULL);
+        MMIO_DH(0xe671c, D_ALL, dpy_reg_mmio_read, NULL);
+        MMIO_DH(0xe681c, D_ALL, dpy_reg_mmio_read, NULL);
+        MMIO_DH(0xe6c04, D_ALL, dpy_reg_mmio_read_2, NULL);
+        MMIO_DH(0xe6e1c, D_ALL, dpy_reg_mmio_read_3, NULL);
+
+        MMIO_RO(PCH_PORT_HOTPLUG, D_ALL, 0,
+                PORTA_HOTPLUG_STATUS_MASK
+                | PORTB_HOTPLUG_STATUS_MASK
+                | PORTC_HOTPLUG_STATUS_MASK
+                | PORTD_HOTPLUG_STATUS_MASK,
+                NULL, NULL);
+
+        MMIO_DH(LCPLL_CTL, D_ALL, NULL, lcpll_ctl_mmio_write);
+        MMIO_D(FUSE_STRAP, D_ALL);
+        MMIO_D(DIGITAL_PORT_HOTPLUG_CNTRL, D_ALL);
+
+        MMIO_D(DISP_ARB_CTL, D_ALL);
+        MMIO_D(DISP_ARB_CTL2, D_ALL);
+
+        MMIO_D(ILK_DISPLAY_CHICKEN1, D_ALL);
+        MMIO_D(ILK_DISPLAY_CHICKEN2, D_ALL);
+        MMIO_D(ILK_DSPCLK_GATE_D, D_ALL);
+
+        MMIO_D(SOUTH_CHICKEN1, D_ALL);
+        MMIO_DH(SOUTH_CHICKEN2, D_ALL, NULL, south_chicken2_mmio_write);
+        MMIO_D(_TRANSA_CHICKEN1, D_ALL);
+        MMIO_D(_TRANSB_CHICKEN1, D_ALL);
+        MMIO_D(SOUTH_DSPCLK_GATE_D, D_ALL);
+        MMIO_D(_TRANSA_CHICKEN2, D_ALL);
+        MMIO_D(_TRANSB_CHICKEN2, D_ALL);
+
+        MMIO_D(ILK_DPFC_CB_BASE, D_ALL);
+        MMIO_D(ILK_DPFC_CONTROL, D_ALL);
+        MMIO_D(ILK_DPFC_RECOMP_CTL, D_ALL);
+        MMIO_D(ILK_DPFC_STATUS, D_ALL);
+        MMIO_D(ILK_DPFC_FENCE_YOFF, D_ALL);
+        MMIO_D(ILK_DPFC_CHICKEN, D_ALL);
+        MMIO_D(ILK_FBC_RT_BASE, D_ALL);
+
+        MMIO_D(IPS_CTL, D_ALL);
+
+        MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BY(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BU(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BV(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_MODE(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_A), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_A), D_ALL);
+
+        MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BY(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BU(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BV(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_MODE(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_B), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_B), D_ALL);
+
+        MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BY(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BU(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_COEFF_BV(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_MODE(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_C), D_ALL);
+        MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_C), D_ALL);
+
+        MMIO_D(PREC_PAL_INDEX(PIPE_A), D_ALL);
+        MMIO_D(PREC_PAL_DATA(PIPE_A), D_ALL);
+        MMIO_F(PREC_PAL_GC_MAX(PIPE_A, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(PREC_PAL_INDEX(PIPE_B), D_ALL);
+        MMIO_D(PREC_PAL_DATA(PIPE_B), D_ALL);
+        MMIO_F(PREC_PAL_GC_MAX(PIPE_B, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(PREC_PAL_INDEX(PIPE_C), D_ALL);
+        MMIO_D(PREC_PAL_DATA(PIPE_C), D_ALL);
+        MMIO_F(PREC_PAL_GC_MAX(PIPE_C, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(0x60110, D_ALL);
+        MMIO_D(0x61110, D_ALL);
+        MMIO_F(0x70400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x71400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x72400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x70440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+        MMIO_F(0x71440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+        MMIO_F(0x72440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+        MMIO_F(0x7044c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+        MMIO_F(0x7144c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+        MMIO_F(0x7244c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+
+        MMIO_D(PIPE_WM_LINETIME(PIPE_A), D_ALL);
+        MMIO_D(PIPE_WM_LINETIME(PIPE_B), D_ALL);
+        MMIO_D(PIPE_WM_LINETIME(PIPE_C), D_ALL);
+        MMIO_D(SPLL_CTL, D_ALL);
+        MMIO_D(_WRPLL_CTL1, D_ALL);
+        MMIO_D(_WRPLL_CTL2, D_ALL);
+        MMIO_D(PORT_CLK_SEL(PORT_A), D_ALL);
+        MMIO_D(PORT_CLK_SEL(PORT_B), D_ALL);
+        MMIO_D(PORT_CLK_SEL(PORT_C), D_ALL);
+        MMIO_D(PORT_CLK_SEL(PORT_D), D_ALL);
+        MMIO_D(PORT_CLK_SEL(PORT_E), D_ALL);
+        MMIO_D(TRANS_CLK_SEL(TRANSCODER_A), D_ALL);
+        MMIO_D(TRANS_CLK_SEL(TRANSCODER_B), D_ALL);
+        MMIO_D(TRANS_CLK_SEL(TRANSCODER_C), D_ALL);
+
+        MMIO_D(HSW_NDE_RSTWRN_OPT, D_ALL);
+        MMIO_D(0x46508, D_ALL);
+
+        MMIO_D(0x49080, D_ALL);
+        MMIO_D(0x49180, D_ALL);
+        MMIO_D(0x49280, D_ALL);
+
+        MMIO_F(0x49090, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x49190, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x49290, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(GAMMA_MODE(PIPE_A), D_ALL);
+        MMIO_D(GAMMA_MODE(PIPE_B), D_ALL);
+        MMIO_D(GAMMA_MODE(PIPE_C), D_ALL);
+
+        MMIO_D(PIPE_MULT(PIPE_A), D_ALL);
+        MMIO_D(PIPE_MULT(PIPE_B), D_ALL);
+        MMIO_D(PIPE_MULT(PIPE_C), D_ALL);
+
+        MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_A), D_ALL);
+        MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_B), D_ALL);
+        MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_C), D_ALL);
+
+        MMIO_DH(SFUSE_STRAP, D_ALL, NULL, NULL);
+        MMIO_D(SBI_ADDR, D_ALL);
+        MMIO_DH(SBI_DATA, D_ALL, sbi_data_mmio_read, NULL);
+        MMIO_DH(SBI_CTL_STAT, D_ALL, NULL, sbi_ctl_mmio_write);
+        MMIO_D(PIXCLK_GATE, D_ALL);
+
+        MMIO_F(_DPA_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_ALL, NULL,
+                dp_aux_ch_ctl_mmio_write);
+
+        MMIO_DH(DDI_BUF_CTL(PORT_A), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+        MMIO_DH(DDI_BUF_CTL(PORT_B), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+        MMIO_DH(DDI_BUF_CTL(PORT_C), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+        MMIO_DH(DDI_BUF_CTL(PORT_D), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+        MMIO_DH(DDI_BUF_CTL(PORT_E), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+
+        MMIO_DH(DP_TP_CTL(PORT_A), D_ALL, NULL, dp_tp_ctl_mmio_write);
+        MMIO_DH(DP_TP_CTL(PORT_B), D_ALL, NULL, dp_tp_ctl_mmio_write);
+        MMIO_DH(DP_TP_CTL(PORT_C), D_ALL, NULL, dp_tp_ctl_mmio_write);
+        MMIO_DH(DP_TP_CTL(PORT_D), D_ALL, NULL, dp_tp_ctl_mmio_write);
+        MMIO_DH(DP_TP_CTL(PORT_E), D_ALL, NULL, dp_tp_ctl_mmio_write);
+
+        MMIO_DH(DP_TP_STATUS(PORT_A), D_ALL, NULL, dp_tp_status_mmio_write);
+        MMIO_DH(DP_TP_STATUS(PORT_B), D_ALL, NULL, dp_tp_status_mmio_write);
+        MMIO_DH(DP_TP_STATUS(PORT_C), D_ALL, NULL, dp_tp_status_mmio_write);
+        MMIO_DH(DP_TP_STATUS(PORT_D), D_ALL, NULL, dp_tp_status_mmio_write);
+        MMIO_DH(DP_TP_STATUS(PORT_E), D_ALL, NULL, NULL);
+
+        MMIO_F(_DDI_BUF_TRANS_A, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x64e60, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x64eC0, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x64f20, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x64f80, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(HSW_AUD_CFG(PIPE_A), D_ALL);
+        MMIO_D(HSW_AUD_PIN_ELD_CP_VLD, D_ALL);
+
+        MMIO_DH(_TRANS_DDI_FUNC_CTL_A, D_ALL, NULL, NULL);
+        MMIO_DH(_TRANS_DDI_FUNC_CTL_B, D_ALL, NULL, NULL);
+        MMIO_DH(_TRANS_DDI_FUNC_CTL_C, D_ALL, NULL, NULL);
+        MMIO_DH(_TRANS_DDI_FUNC_CTL_EDP, D_ALL, NULL, NULL);
+
+        MMIO_D(_TRANSA_MSA_MISC, D_ALL);
+        MMIO_D(_TRANSB_MSA_MISC, D_ALL);
+        MMIO_D(_TRANSC_MSA_MISC, D_ALL);
+        MMIO_D(_TRANS_EDP_MSA_MISC, D_ALL);
+
+        MMIO_DH(FORCEWAKE, D_ALL, NULL, NULL);
+        MMIO_D(FORCEWAKE_ACK, D_ALL);
+        MMIO_D(GEN6_GT_CORE_STATUS, D_ALL);
+        MMIO_D(GEN6_GT_THREAD_STATUS_REG, D_ALL);
+        MMIO_D(GTFIFODBG, D_ALL);
+        MMIO_D(GTFIFOCTL, D_ALL);
+        MMIO_DH(FORCEWAKE_MT, D_PRE_SKL, NULL, mul_force_wake_write);
+        MMIO_DH(FORCEWAKE_ACK_HSW, D_HSW | D_BDW, NULL, NULL);
+        MMIO_D(ECOBUS, D_ALL);
+        MMIO_DH(GEN6_RC_CONTROL, D_ALL, NULL, NULL);
+        MMIO_DH(GEN6_RC_STATE, D_ALL, NULL, NULL);
+        MMIO_D(GEN6_RPNSWREQ, D_ALL);
+        MMIO_D(GEN6_RC_VIDEO_FREQ, D_ALL);
+        MMIO_D(GEN6_RP_DOWN_TIMEOUT, D_ALL);
+        MMIO_D(GEN6_RP_INTERRUPT_LIMITS, D_ALL);
+        MMIO_D(GEN6_RPSTAT1, D_ALL);
+        MMIO_D(GEN6_RP_CONTROL, D_ALL);
+        MMIO_D(GEN6_RP_UP_THRESHOLD, D_ALL);
+        MMIO_D(GEN6_RP_DOWN_THRESHOLD, D_ALL);
+        MMIO_D(GEN6_RP_CUR_UP_EI, D_ALL);
+        MMIO_D(GEN6_RP_CUR_UP, D_ALL);
+        MMIO_D(GEN6_RP_PREV_UP, D_ALL);
+        MMIO_D(GEN6_RP_CUR_DOWN_EI, D_ALL);
+        MMIO_D(GEN6_RP_CUR_DOWN, D_ALL);
+        MMIO_D(GEN6_RP_PREV_DOWN, D_ALL);
+        MMIO_D(GEN6_RP_UP_EI, D_ALL);
+        MMIO_D(GEN6_RP_DOWN_EI, D_ALL);
+        MMIO_D(GEN6_RP_IDLE_HYSTERSIS, D_ALL);
+        MMIO_D(GEN6_RC1_WAKE_RATE_LIMIT, D_ALL);
+        MMIO_D(GEN6_RC6_WAKE_RATE_LIMIT, D_ALL);
+        MMIO_D(GEN6_RC6pp_WAKE_RATE_LIMIT, D_ALL);
+        MMIO_D(GEN6_RC_EVALUATION_INTERVAL, D_ALL);
+        MMIO_D(GEN6_RC_IDLE_HYSTERSIS, D_ALL);
+        MMIO_D(GEN6_RC_SLEEP, D_ALL);
+        MMIO_D(GEN6_RC1e_THRESHOLD, D_ALL);
+        MMIO_D(GEN6_RC6_THRESHOLD, D_ALL);
+        MMIO_D(GEN6_RC6p_THRESHOLD, D_ALL);
+        MMIO_D(GEN6_RC6pp_THRESHOLD, D_ALL);
+        MMIO_D(GEN6_PMINTRMSK, D_ALL);
+        MMIO_DH(HSW_PWR_WELL_BIOS, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+        MMIO_DH(HSW_PWR_WELL_DRIVER, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+        MMIO_DH(HSW_PWR_WELL_KVMR, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+        MMIO_DH(HSW_PWR_WELL_DEBUG, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+        MMIO_DH(HSW_PWR_WELL_CTL5, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+        MMIO_DH(HSW_PWR_WELL_CTL6, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+
+        MMIO_D(RSTDBYCTL, D_ALL);
+
+        MMIO_DH(GEN6_GDRST, D_ALL, NULL, gdrst_mmio_write);
+        MMIO_F(FENCE_REG_GEN6_LO(0), 0x80, 0, 0, 0, D_ALL, fence_mmio_read, fence_mmio_write);
+        MMIO_F(VGT_PVINFO_PAGE, VGT_PVINFO_SIZE, F_UNALIGN, 0, 0, D_ALL, pvinfo_mmio_read, pvinfo_mmio_write);
+        MMIO_DH(CPU_VGACNTRL, D_ALL, NULL, vga_control_mmio_write);
+
+        MMIO_F(MCHBAR_MIRROR_BASE_SNB, 0x40000, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(TILECTL, D_ALL);
+
+        MMIO_D(GEN6_UCGCTL1, D_ALL);
+        MMIO_D(GEN6_UCGCTL2, D_ALL);
+
+        MMIO_F(0x4f000, 0x90, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_D(GEN6_PCODE_MAILBOX, D_PRE_SKL);
+        MMIO_D(GEN6_PCODE_DATA, D_ALL);
+        MMIO_D(0x13812c, D_ALL);
+        MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL);
+        MMIO_D(HSW_EDRAM_CAP, D_ALL);
+        MMIO_D(HSW_IDICR, D_ALL);
+        MMIO_DH(GFX_FLSH_CNTL_GEN6, D_ALL, NULL, NULL);
+
+        MMIO_D(0x3c, D_ALL);
+        MMIO_D(0x860, D_ALL);
+        MMIO_D(ECOSKPD, D_ALL);
+        MMIO_D(0x121d0, D_ALL);
+        MMIO_D(GEN6_BLITTER_ECOSKPD, D_ALL);
+        MMIO_D(0x41d0, D_ALL);
+        MMIO_D(GAC_ECO_BITS, D_ALL);
+        MMIO_D(0x6200, D_ALL);
+        MMIO_D(0x6204, D_ALL);
+        MMIO_D(0x6208, D_ALL);
+        MMIO_D(0x7118, D_ALL);
+        MMIO_D(0x7180, D_ALL);
+        MMIO_D(0x7408, D_ALL);
+        MMIO_D(0x7c00, D_ALL);
+        MMIO_D(GEN6_MBCTL, D_ALL);
+        MMIO_D(0x911c, D_ALL);
+        MMIO_D(0x9120, D_ALL);
+
+        MMIO_D(GAB_CTL, D_ALL);
+        MMIO_D(0x48800, D_ALL);
+        MMIO_D(0xce044, D_ALL);
+        MMIO_D(0xe6500, D_ALL);
+        MMIO_D(0xe6504, D_ALL);
+        MMIO_D(0xe6600, D_ALL);
+        MMIO_D(0xe6604, D_ALL);
+        MMIO_D(0xe6700, D_ALL);
+        MMIO_D(0xe6704, D_ALL);
+        MMIO_D(0xe6800, D_ALL);
+        MMIO_D(0xe6804, D_ALL);
+        MMIO_D(PCH_GMBUS4, D_ALL);
+        MMIO_D(PCH_GMBUS5, D_ALL);
+
+        MMIO_D(0x902c, D_ALL);
+        MMIO_D(0xec008, D_ALL);
+        MMIO_D(0xec00c, D_ALL);
+        MMIO_D(0xec008 + 0x18, D_ALL);
+        MMIO_D(0xec00c + 0x18, D_ALL);
+        MMIO_D(0xec008 + 0x18 * 2, D_ALL);
+        MMIO_D(0xec00c + 0x18 * 2, D_ALL);
+        MMIO_D(0xec008 + 0x18 * 3, D_ALL);
+        MMIO_D(0xec00c + 0x18 * 3, D_ALL);
+        MMIO_D(0xec408, D_ALL);
+        MMIO_D(0xec40c, D_ALL);
+        MMIO_D(0xec408 + 0x18, D_ALL);
+        MMIO_D(0xec40c + 0x18, D_ALL);
+        MMIO_D(0xec408 + 0x18 * 2, D_ALL);
+        MMIO_D(0xec40c + 0x18 * 2, D_ALL);
+        MMIO_D(0xec408 + 0x18 * 3, D_ALL);
+        MMIO_D(0xec40c + 0x18 * 3, D_ALL);
+        MMIO_D(0xfc810, D_ALL);
+        MMIO_D(0xfc81c, D_ALL);
+        MMIO_D(0xfc828, D_ALL);
+        MMIO_D(0xfc834, D_ALL);
+        MMIO_D(0xfcc00, D_ALL);
+        MMIO_D(0xfcc0c, D_ALL);
+        MMIO_D(0xfcc18, D_ALL);
+        MMIO_D(0xfcc24, D_ALL);
+        MMIO_D(0xfd000, D_ALL);
+        MMIO_D(0xfd00c, D_ALL);
+        MMIO_D(0xfd018, D_ALL);
+        MMIO_D(0xfd024, D_ALL);
+        MMIO_D(0xfd034, D_ALL);
+
+        MMIO_DH(FPGA_DBG, D_ALL, NULL, fpga_dbg_mmio_write);
+        MMIO_D(0x2054, D_ALL);
+        MMIO_D(0x12054, D_ALL);
+        MMIO_D(0x22054, D_ALL);
+        MMIO_D(0x1a054, D_ALL);
+
+        MMIO_D(0x44070, D_ALL);
+
+        MMIO_D(0x215c, D_HSW_PLUS);
+        MMIO_DFH(0x2178, D_ALL, F_CMD_ACCESS, NULL, NULL);
+        MMIO_DFH(0x217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
+        MMIO_DFH(0x12178, D_ALL, F_CMD_ACCESS, NULL, NULL);
+        MMIO_DFH(0x1217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
+
+        MMIO_F(0x2290, 8, 0, 0, 0, D_HSW_PLUS, NULL, NULL);
+        MMIO_D(OACONTROL, D_HSW);
+        MMIO_D(0x2b00, D_BDW_PLUS);
+        MMIO_D(0x2360, D_BDW_PLUS);
+        MMIO_F(0x5200, 32, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x5240, 32, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(0x5280, 16, 0, 0, 0, D_ALL, NULL, NULL);
+
+        MMIO_DFH(0x1c17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+        MMIO_DFH(0x1c178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+        MMIO_D(BCS_SWCTRL, D_ALL);
+
+        MMIO_F(HS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(DS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(IA_VERTICES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(IA_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(VS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(GS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(GS_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(CL_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(CL_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(PS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_F(PS_DEPTH_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+        MMIO_DH(0x4260, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+        MMIO_DH(0x4264, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+        MMIO_DH(0x4268, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+        MMIO_DH(0x426c, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+        MMIO_DH(0x4270, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+        MMIO_DFH(0x4094, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+
+        return 0;
+}
+
+static int init_broadwell_mmio_info(struct intel_gvt *gvt)
+{
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        int ret;
+
+        MMIO_DH(RING_IMR(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL,
+                        intel_vgpu_reg_imr_handler);
+
+        MMIO_DH(GEN8_GT_IMR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_GT_IER(0), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_GT_IIR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_GT_ISR(0), D_BDW_PLUS);
+
+        MMIO_DH(GEN8_GT_IMR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_GT_IER(1), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_GT_IIR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_GT_ISR(1), D_BDW_PLUS);
+
+        MMIO_DH(GEN8_GT_IMR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_GT_IER(2), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_GT_IIR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_GT_ISR(2), D_BDW_PLUS);
+
+        MMIO_DH(GEN8_GT_IMR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_GT_IER(3), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_GT_IIR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_GT_ISR(3), D_BDW_PLUS);
+
+        MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_A), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_DE_PIPE_IER(PIPE_A), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_A), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_DE_PIPE_ISR(PIPE_A), D_BDW_PLUS);
+
+        MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_B), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_DE_PIPE_IER(PIPE_B), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_B), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_DE_PIPE_ISR(PIPE_B), D_BDW_PLUS);
+
+        MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_C), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_DE_PIPE_IER(PIPE_C), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_C), D_BDW_PLUS, NULL,
+                intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_DE_PIPE_ISR(PIPE_C), D_BDW_PLUS);
+
+        MMIO_DH(GEN8_DE_PORT_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_DE_PORT_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_DE_PORT_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_DE_PORT_ISR, D_BDW_PLUS);
+
+        MMIO_DH(GEN8_DE_MISC_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_DE_MISC_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_DE_MISC_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_DE_MISC_ISR, D_BDW_PLUS);
+
+        MMIO_DH(GEN8_PCU_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+        MMIO_DH(GEN8_PCU_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+        MMIO_DH(GEN8_PCU_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+        MMIO_D(GEN8_PCU_ISR, D_BDW_PLUS);
+
+        MMIO_DH(GEN8_MASTER_IRQ, D_BDW_PLUS, NULL,
+                intel_vgpu_reg_master_irq_handler);
+
+        MMIO_D(RING_HWSTAM(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+        MMIO_D(0x1c134, D_BDW_PLUS);
+
+        MMIO_D(RING_TAIL(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+        MMIO_D(RING_HEAD(GEN8_BSD2_RING_BASE),  D_BDW_PLUS);
+        MMIO_GM(RING_START(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, NULL);
+        MMIO_D(RING_CTL(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+        MMIO_D(RING_ACTHD(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+        MMIO_D(RING_ACTHD_UDW(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+        MMIO_DFH(0x1c29c, D_BDW_PLUS, F_MODE_MASK, NULL, ring_mode_mmio_write);
+        MMIO_DFH(RING_MI_MODE(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK,
+                        NULL, NULL);
+        MMIO_DFH(RING_INSTPM(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK,
+                        NULL, NULL);
+        MMIO_DFH(RING_TIMESTAMP(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_CMD_ACCESS,
+                        ring_timestamp_mmio_read, NULL);
+
+        MMIO_RING_D(RING_ACTHD_UDW, D_BDW_PLUS);
+
+#define RING_REG(base) (base + 0x230)
+        MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write);
+        MMIO_DH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, elsp_mmio_write);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x234)
+        MMIO_RING_F(RING_REG, 8, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL);
+        MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 4, F_RO, 0, ~0LL, D_BDW_PLUS, NULL, NULL);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x244)
+        MMIO_RING_D(RING_REG, D_BDW_PLUS);
+        MMIO_D(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x370)
+        MMIO_RING_F(RING_REG, 48, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL);
+        MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 48, F_RO, 0, ~0, D_BDW_PLUS,
+                        NULL, NULL);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x3a0)
+        MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+#undef RING_REG
+
+        MMIO_D(PIPEMISC(PIPE_A), D_BDW_PLUS);
+        MMIO_D(PIPEMISC(PIPE_B), D_BDW_PLUS);
+        MMIO_D(PIPEMISC(PIPE_C), D_BDW_PLUS);
+        MMIO_D(0x1c1d0, D_BDW_PLUS);
+        MMIO_D(GEN6_MBCUNIT_SNPCR, D_BDW_PLUS);
+        MMIO_D(GEN7_MISCCPCTL, D_BDW_PLUS);
+        MMIO_D(0x1c054, D_BDW_PLUS);
+
+        MMIO_D(GEN8_PRIVATE_PAT_LO, D_BDW_PLUS);
+        MMIO_D(GEN8_PRIVATE_PAT_HI, D_BDW_PLUS);
+
+        MMIO_D(GAMTARBMODE, D_BDW_PLUS);
+
+#define RING_REG(base) (base + 0x270)
+        MMIO_RING_F(RING_REG, 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL);
+        MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL);
+#undef RING_REG
+
+        MMIO_RING_GM(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL);
+        MMIO_GM(0x1c080, D_BDW_PLUS, NULL, NULL);
+
+        MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+
+        MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW);
+        MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW);
+        MMIO_D(CHICKEN_PIPESL_1(PIPE_C), D_BDW);
+
+        MMIO_D(WM_MISC, D_BDW);
+        MMIO_D(BDW_EDP_PSR_BASE, D_BDW);
+
+        MMIO_D(0x66c00, D_BDW_PLUS);
+        MMIO_D(0x66c04, D_BDW_PLUS);
+
+        MMIO_D(HSW_GTT_CACHE_EN, D_BDW_PLUS);
+
+        MMIO_D(GEN8_EU_DISABLE0, D_BDW_PLUS);
+        MMIO_D(GEN8_EU_DISABLE1, D_BDW_PLUS);
+        MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS);
+
+        MMIO_D(0xfdc, D_BDW);
+        MMIO_D(GEN8_ROW_CHICKEN, D_BDW_PLUS);
+        MMIO_D(GEN7_ROW_CHICKEN2, D_BDW_PLUS);
+        MMIO_D(GEN8_UCGCTL6, D_BDW_PLUS);
+
+        MMIO_D(0xb1f0, D_BDW);
+        MMIO_D(0xb1c0, D_BDW);
+        MMIO_DFH(GEN8_L3SQCREG4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+        MMIO_D(0xb100, D_BDW);
+        MMIO_D(0xb10c, D_BDW);
+        MMIO_D(0xb110, D_BDW);
+
+        MMIO_DH(0x24d0, D_BDW_PLUS, NULL, NULL);
+        MMIO_DH(0x24d4, D_BDW_PLUS, NULL, NULL);
+        MMIO_DH(0x24d8, D_BDW_PLUS, NULL, NULL);
+        MMIO_DH(0x24dc, D_BDW_PLUS, NULL, NULL);
+
+        MMIO_D(0x83a4, D_BDW);
+        MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS);
+
+        MMIO_D(0x8430, D_BDW);
+
+        MMIO_D(0x110000, D_BDW_PLUS);
+
+        MMIO_D(0x48400, D_BDW_PLUS);
+
+        MMIO_D(0x6e570, D_BDW_PLUS);
+        MMIO_D(0x65f10, D_BDW_PLUS);
+
+        MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0xe180, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+        MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+
+        MMIO_D(0x2248, D_BDW);
+
+        return 0;
+}
+
+static int init_skl_mmio_info(struct intel_gvt *gvt)
+{
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        int ret;
+
+        MMIO_DH(FORCEWAKE_RENDER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
+        MMIO_DH(FORCEWAKE_ACK_RENDER_GEN9, D_SKL_PLUS, NULL, NULL);
+        MMIO_DH(FORCEWAKE_BLITTER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
+        MMIO_DH(FORCEWAKE_ACK_BLITTER_GEN9, D_SKL_PLUS, NULL, NULL);
+        MMIO_DH(FORCEWAKE_MEDIA_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
+        MMIO_DH(FORCEWAKE_ACK_MEDIA_GEN9, D_SKL_PLUS, NULL, NULL);
+
+        MMIO_F(_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL, NULL, dp_aux_ch_ctl_mmio_write);
+        MMIO_F(_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL, NULL, dp_aux_ch_ctl_mmio_write);
+        MMIO_F(_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL, NULL, dp_aux_ch_ctl_mmio_write);
+
+        MMIO_D(HSW_PWR_WELL_BIOS, D_SKL);
+        MMIO_DH(HSW_PWR_WELL_DRIVER, D_SKL, NULL, skl_power_well_ctl_write);
+
+        MMIO_DH(GEN6_PCODE_MAILBOX, D_SKL, NULL, mailbox_write);
+        MMIO_D(0xa210, D_SKL_PLUS);
+        MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
+        MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
+        MMIO_DH(0x4ddc, D_SKL, NULL, skl_misc_ctl_write);
+        MMIO_DH(0x42080, D_SKL, NULL, skl_misc_ctl_write);
+        MMIO_D(0x45504, D_SKL);
+        MMIO_D(0x45520, D_SKL);
+        MMIO_D(0x46000, D_SKL);
+        MMIO_DH(0x46010, D_SKL, NULL, skl_lcpll_write);
+        MMIO_DH(0x46014, D_SKL, NULL, skl_lcpll_write);
+        MMIO_D(0x6C040, D_SKL);
+        MMIO_D(0x6C048, D_SKL);
+        MMIO_D(0x6C050, D_SKL);
+        MMIO_D(0x6C044, D_SKL);
+        MMIO_D(0x6C04C, D_SKL);
+        MMIO_D(0x6C054, D_SKL);
+        MMIO_D(0x6c058, D_SKL);
+        MMIO_D(0x6c05c, D_SKL);
+        MMIO_DH(0X6c060, D_SKL, dpll_status_read, NULL);
+
+        MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 1), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 1), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 1), D_SKL, NULL, pf_write);
+
+        MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 1), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 1), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 1), D_SKL, NULL, pf_write);
+
+        MMIO_DH(SKL_PS_CTRL(PIPE_A, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_CTRL(PIPE_A, 1), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_CTRL(PIPE_B, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_CTRL(PIPE_B, 1), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_CTRL(PIPE_C, 0), D_SKL, NULL, pf_write);
+        MMIO_DH(SKL_PS_CTRL(PIPE_C, 1), D_SKL, NULL, pf_write);
+
+        MMIO_DH(PLANE_BUF_CFG(PIPE_A, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_A, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_A, 2), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_A, 3), D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_BUF_CFG(PIPE_B, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_B, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_B, 2), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_B, 3), D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_BUF_CFG(PIPE_C, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_C, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_C, 2), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_BUF_CFG(PIPE_C, 3), D_SKL, NULL, NULL);
+
+        MMIO_DH(CUR_BUF_CFG(PIPE_A), D_SKL, NULL, NULL);
+        MMIO_DH(CUR_BUF_CFG(PIPE_B), D_SKL, NULL, NULL);
+        MMIO_DH(CUR_BUF_CFG(PIPE_C), D_SKL, NULL, NULL);
+
+        MMIO_F(PLANE_WM(PIPE_A, 0, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(PLANE_WM(PIPE_A, 1, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(PLANE_WM(PIPE_A, 2, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+        MMIO_F(PLANE_WM(PIPE_B, 0, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(PLANE_WM(PIPE_B, 1, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(PLANE_WM(PIPE_B, 2, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+        MMIO_F(PLANE_WM(PIPE_C, 0, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(PLANE_WM(PIPE_C, 1, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(PLANE_WM(PIPE_C, 2, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+        MMIO_F(CUR_WM(PIPE_A, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(CUR_WM(PIPE_B, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(CUR_WM(PIPE_C, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_WM_TRANS(PIPE_A, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_WM_TRANS(PIPE_A, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_WM_TRANS(PIPE_A, 2), D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_WM_TRANS(PIPE_B, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_WM_TRANS(PIPE_B, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_WM_TRANS(PIPE_B, 2), D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_WM_TRANS(PIPE_C, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_WM_TRANS(PIPE_C, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_WM_TRANS(PIPE_C, 2), D_SKL, NULL, NULL);
+
+        MMIO_DH(CUR_WM_TRANS(PIPE_A), D_SKL, NULL, NULL);
+        MMIO_DH(CUR_WM_TRANS(PIPE_B), D_SKL, NULL, NULL);
+        MMIO_DH(CUR_WM_TRANS(PIPE_C), D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 2), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 3), D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 2), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 3), D_SKL, NULL, NULL);
+
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 0), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 1), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 2), D_SKL, NULL, NULL);
+        MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 3), D_SKL, NULL, NULL);
+
+        MMIO_DH(_REG_701C0(PIPE_A, 1), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_A, 2), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_A, 3), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_A, 4), D_SKL, NULL, NULL);
+
+        MMIO_DH(_REG_701C0(PIPE_B, 1), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_B, 2), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_B, 3), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_B, 4), D_SKL, NULL, NULL);
+
+        MMIO_DH(_REG_701C0(PIPE_C, 1), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_C, 2), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_C, 3), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C0(PIPE_C, 4), D_SKL, NULL, NULL);
+
+        MMIO_DH(_REG_701C4(PIPE_A, 1), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_A, 2), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_A, 3), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_A, 4), D_SKL, NULL, NULL);
+
+        MMIO_DH(_REG_701C4(PIPE_B, 1), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_B, 2), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_B, 3), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_B, 4), D_SKL, NULL, NULL);
+
+        MMIO_DH(_REG_701C4(PIPE_C, 1), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_C, 2), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_C, 3), D_SKL, NULL, NULL);
+        MMIO_DH(_REG_701C4(PIPE_C, 4), D_SKL, NULL, NULL);
+
+        MMIO_D(0x70380, D_SKL);
+        MMIO_D(0x71380, D_SKL);
+        MMIO_D(0x72380, D_SKL);
+        MMIO_D(0x7039c, D_SKL);
+
+        MMIO_F(0x80000, 0x3000, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_D(0x8f074, D_SKL);
+        MMIO_D(0x8f004, D_SKL);
+        MMIO_D(0x8f034, D_SKL);
+
+        MMIO_D(0xb11c, D_SKL);
+
+        MMIO_D(0x51000, D_SKL);
+        MMIO_D(0x6c00c, D_SKL);
+
+        MMIO_F(0xc800, 0x7f8, 0, 0, 0, D_SKL, NULL, NULL);
+        MMIO_F(0xb020, 0x80, 0, 0, 0, D_SKL, NULL, NULL);
+
+        MMIO_D(0xd08, D_SKL);
+        MMIO_D(0x20e0, D_SKL);
+        MMIO_D(0x20ec, D_SKL);
+
+        /* TRTT */
+        MMIO_D(0x4de0, D_SKL);
+        MMIO_D(0x4de4, D_SKL);
+        MMIO_D(0x4de8, D_SKL);
+        MMIO_D(0x4dec, D_SKL);
+        MMIO_D(0x4df0, D_SKL);
+        MMIO_DH(0x4df4, D_SKL, NULL, gen9_trtte_write);
+        MMIO_DH(0x4dfc, D_SKL, NULL, gen9_trtt_chicken_write);
+
+        MMIO_D(0x45008, D_SKL);
+
+        MMIO_D(0x46430, D_SKL);
+
+        MMIO_D(0x46520, D_SKL);
+
+        MMIO_D(0xc403c, D_SKL);
+        MMIO_D(0xb004, D_SKL);
+        MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write);
+
+        MMIO_D(0x65900, D_SKL);
+        MMIO_D(0x1082c0, D_SKL);
+        MMIO_D(0x4068, D_SKL);
+        MMIO_D(0x67054, D_SKL);
+        MMIO_D(0x6e560, D_SKL);
+        MMIO_D(0x6e554, D_SKL);
+        MMIO_D(0x2b20, D_SKL);
+        MMIO_D(0x65f00, D_SKL);
+        MMIO_D(0x65f08, D_SKL);
+        MMIO_D(0x320f0, D_SKL);
+
+        MMIO_D(_REG_VCS2_EXCC, D_SKL);
+        MMIO_D(0x70034, D_SKL);
+        MMIO_D(0x71034, D_SKL);
+        MMIO_D(0x72034, D_SKL);
+
+        MMIO_D(_PLANE_KEYVAL_1(PIPE_A), D_SKL);
+        MMIO_D(_PLANE_KEYVAL_1(PIPE_B), D_SKL);
+        MMIO_D(_PLANE_KEYVAL_1(PIPE_C), D_SKL);
+        MMIO_D(_PLANE_KEYMSK_1(PIPE_A), D_SKL);
+        MMIO_D(_PLANE_KEYMSK_1(PIPE_B), D_SKL);
+        MMIO_D(_PLANE_KEYMSK_1(PIPE_C), D_SKL);
+
+        MMIO_D(0x44500, D_SKL);
+        return 0;
+}
+
+/**
+ * intel_gvt_find_mmio_info - find MMIO information entry by aligned offset
+ * @gvt: GVT device
+ * @offset: register offset
+ *
+ * This function is used to find the MMIO information entry from hash table
+ *
+ * Returns:
+ * pointer to MMIO information entry, NULL if not exists
+ */
+struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
+        unsigned int offset)
+{
+        struct intel_gvt_mmio_info *e;
+
+        WARN_ON(!IS_ALIGNED(offset, 4));
+
+        hash_for_each_possible(gvt->mmio.mmio_info_table, e, node, offset) {
+                if (e->offset == offset)
+                        return e;
+        }
+        return NULL;
+}
+
+/**
+ * intel_gvt_clean_mmio_info - clean up MMIO information table for GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the driver unloading stage, to clean up the MMIO
+ * information table of GVT device
+ *
+ */
+void intel_gvt_clean_mmio_info(struct intel_gvt *gvt)
+{
+        struct hlist_node *tmp;
+        struct intel_gvt_mmio_info *e;
+        int i;
+
+        hash_for_each_safe(gvt->mmio.mmio_info_table, i, tmp, e, node)
+                kfree(e);
+
+        vfree(gvt->mmio.mmio_attribute);
+        gvt->mmio.mmio_attribute = NULL;
+}
+
+/**
+ * intel_gvt_setup_mmio_info - setup MMIO information table for GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the initialization stage, to setup the MMIO
+ * information table for GVT device
+ *
+ * Returns:
+ * zero on success, negative if failed.
+ */
+int intel_gvt_setup_mmio_info(struct intel_gvt *gvt)
+{
+        struct intel_gvt_device_info *info = &gvt->device_info;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        int ret;
+
+        gvt->mmio.mmio_attribute = vzalloc(info->mmio_size);
+        if (!gvt->mmio.mmio_attribute)
+                return -ENOMEM;
+
+        ret = init_generic_mmio_info(gvt);
+        if (ret)
+                goto err;
+
+        if (IS_BROADWELL(dev_priv)) {
+                ret = init_broadwell_mmio_info(gvt);
+                if (ret)
+                        goto err;
+        } else if (IS_SKYLAKE(dev_priv)) {
+                ret = init_broadwell_mmio_info(gvt);
+                if (ret)
+                        goto err;
+                ret = init_skl_mmio_info(gvt);
+                if (ret)
+                        goto err;
+        }
+        return 0;
+err:
+        intel_gvt_clean_mmio_info(gvt);
+        return ret;
+}
+
+/**
+ * intel_gvt_mmio_set_accessed - mark a MMIO has been accessed
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+void intel_gvt_mmio_set_accessed(struct intel_gvt *gvt, unsigned int offset)
+{
+        gvt->mmio.mmio_attribute[offset >> 2] |=
+                F_ACCESSED;
+}
+
+/**
+ * intel_gvt_mmio_is_cmd_accessed - mark a MMIO could be accessed by command
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt,
+                unsigned int offset)
+{
+        return gvt->mmio.mmio_attribute[offset >> 2] &
+                F_CMD_ACCESS;
+}
+
+/**
+ * intel_gvt_mmio_is_unalign - mark a MMIO could be accessed unaligned
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt,
+                unsigned int offset)
+{
+        return gvt->mmio.mmio_attribute[offset >> 2] &
+                F_UNALIGN;
+}
+
+/**
+ * intel_gvt_mmio_set_cmd_accessed - mark a MMIO has been accessed by command
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+void intel_gvt_mmio_set_cmd_accessed(struct intel_gvt *gvt,
+                unsigned int offset)
+{
+        gvt->mmio.mmio_attribute[offset >> 2] |=
+                F_CMD_ACCESSED;
+}
+
+/**
+ * intel_gvt_mmio_has_mode_mask - if a MMIO has a mode mask
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ * Returns:
+ * True if a MMIO has a mode mask in its higher 16 bits, false if it isn't.
+ *
+ */
+bool intel_gvt_mmio_has_mode_mask(struct intel_gvt *gvt, unsigned int offset)
+{
+        return gvt->mmio.mmio_attribute[offset >> 2] &
+                F_MODE_MASK;
+}
+
+/**
+ * intel_vgpu_default_mmio_read - default MMIO read handler
+ * @vgpu: a vGPU
+ * @offset: access offset
+ * @p_data: data return buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        read_vreg(vgpu, offset, p_data, bytes);
+        return 0;
+}
+
+/**
+ * intel_t_default_mmio_write - default MMIO write handler
+ * @vgpu: a vGPU
+ * @offset: access offset
+ * @p_data: write data buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        write_vreg(vgpu, offset, p_data, bytes);
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h
index 254df8bf1f35..027ef558d91c 100644
--- a/drivers/gpu/drm/i915/gvt/hypercall.h
+++ b/drivers/gpu/drm/i915/gvt/hypercall.h
@@ -19,17 +19,51 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Dexuan Cui
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_HYPERCALL_H_
 #define _GVT_HYPERCALL_H_
 
+struct intel_gvt_io_emulation_ops {
+        int (*emulate_cfg_read)(void *, unsigned int, void *, unsigned int);
+        int (*emulate_cfg_write)(void *, unsigned int, void *, unsigned int);
+        int (*emulate_mmio_read)(void *, u64, void *, unsigned int);
+        int (*emulate_mmio_write)(void *, u64, void *, unsigned int);
+};
+
+extern struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops;
+
 /*
  * Specific GVT-g MPT modules function collections. Currently GVT-g supports
  * both Xen and KVM by providing dedicated hypervisor-related MPT modules.
  */
 struct intel_gvt_mpt {
         int (*detect_host)(void);
+        int (*attach_vgpu)(void *vgpu, unsigned long *handle);
+        void (*detach_vgpu)(unsigned long handle);
+        int (*inject_msi)(unsigned long handle, u32 addr, u16 data);
+        unsigned long (*from_virt_to_mfn)(void *p);
+        int (*set_wp_page)(unsigned long handle, u64 gfn);
+        int (*unset_wp_page)(unsigned long handle, u64 gfn);
+        int (*read_gpa)(unsigned long handle, unsigned long gpa, void *buf,
+                        unsigned long len);
+        int (*write_gpa)(unsigned long handle, unsigned long gpa, void *buf,
+                         unsigned long len);
+        unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn);
+        int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn,
+                              unsigned long mfn, unsigned int nr, bool map,
+                              int type);
+        int (*set_trap_area)(unsigned long handle, u64 start, u64 end,
+                             bool map);
 };
 
 extern struct intel_gvt_mpt xengt_mpt;
diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c
new file mode 100644
index 000000000000..f7be02ac4be1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/interrupt.c
@@ -0,0 +1,741 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min he <min.he@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+/* common offset among interrupt control registers */
+#define regbase_to_isr(base)    (base)
+#define regbase_to_imr(base)    (base + 0x4)
+#define regbase_to_iir(base)    (base + 0x8)
+#define regbase_to_ier(base)    (base + 0xC)
+
+#define iir_to_regbase(iir)    (iir - 0x8)
+#define ier_to_regbase(ier)    (ier - 0xC)
+
+#define get_event_virt_handler(irq, e)  (irq->events[e].v_handler)
+#define get_irq_info(irq, e)            (irq->events[e].info)
+
+#define irq_to_gvt(irq) \
+        container_of(irq, struct intel_gvt, irq)
+
+static void update_upstream_irq(struct intel_vgpu *vgpu,
+                struct intel_gvt_irq_info *info);
+
+static const char * const irq_name[INTEL_GVT_EVENT_MAX] = {
+        [RCS_MI_USER_INTERRUPT] = "Render CS MI USER INTERRUPT",
+        [RCS_DEBUG] = "Render EU debug from SVG",
+        [RCS_MMIO_SYNC_FLUSH] = "Render MMIO sync flush status",
+        [RCS_CMD_STREAMER_ERR] = "Render CS error interrupt",
+        [RCS_PIPE_CONTROL] = "Render PIPE CONTROL notify",
+        [RCS_WATCHDOG_EXCEEDED] = "Render CS Watchdog counter exceeded",
+        [RCS_PAGE_DIRECTORY_FAULT] = "Render page directory faults",
+        [RCS_AS_CONTEXT_SWITCH] = "Render AS Context Switch Interrupt",
+
+        [VCS_MI_USER_INTERRUPT] = "Video CS MI USER INTERRUPT",
+        [VCS_MMIO_SYNC_FLUSH] = "Video MMIO sync flush status",
+        [VCS_CMD_STREAMER_ERR] = "Video CS error interrupt",
+        [VCS_MI_FLUSH_DW] = "Video MI FLUSH DW notify",
+        [VCS_WATCHDOG_EXCEEDED] = "Video CS Watchdog counter exceeded",
+        [VCS_PAGE_DIRECTORY_FAULT] = "Video page directory faults",
+        [VCS_AS_CONTEXT_SWITCH] = "Video AS Context Switch Interrupt",
+        [VCS2_MI_USER_INTERRUPT] = "VCS2 Video CS MI USER INTERRUPT",
+        [VCS2_MI_FLUSH_DW] = "VCS2 Video MI FLUSH DW notify",
+        [VCS2_AS_CONTEXT_SWITCH] = "VCS2 Context Switch Interrupt",
+
+        [BCS_MI_USER_INTERRUPT] = "Blitter CS MI USER INTERRUPT",
+        [BCS_MMIO_SYNC_FLUSH] = "Billter MMIO sync flush status",
+        [BCS_CMD_STREAMER_ERR] = "Blitter CS error interrupt",
+        [BCS_MI_FLUSH_DW] = "Blitter MI FLUSH DW notify",
+        [BCS_PAGE_DIRECTORY_FAULT] = "Blitter page directory faults",
+        [BCS_AS_CONTEXT_SWITCH] = "Blitter AS Context Switch Interrupt",
+
+        [VECS_MI_FLUSH_DW] = "Video Enhanced Streamer MI FLUSH DW notify",
+        [VECS_AS_CONTEXT_SWITCH] = "VECS Context Switch Interrupt",
+
+        [PIPE_A_FIFO_UNDERRUN] = "Pipe A FIFO underrun",
+        [PIPE_A_CRC_ERR] = "Pipe A CRC error",
+        [PIPE_A_CRC_DONE] = "Pipe A CRC done",
+        [PIPE_A_VSYNC] = "Pipe A vsync",
+        [PIPE_A_LINE_COMPARE] = "Pipe A line compare",
+        [PIPE_A_ODD_FIELD] = "Pipe A odd field",
+        [PIPE_A_EVEN_FIELD] = "Pipe A even field",
+        [PIPE_A_VBLANK] = "Pipe A vblank",
+        [PIPE_B_FIFO_UNDERRUN] = "Pipe B FIFO underrun",
+        [PIPE_B_CRC_ERR] = "Pipe B CRC error",
+        [PIPE_B_CRC_DONE] = "Pipe B CRC done",
+        [PIPE_B_VSYNC] = "Pipe B vsync",
+        [PIPE_B_LINE_COMPARE] = "Pipe B line compare",
+        [PIPE_B_ODD_FIELD] = "Pipe B odd field",
+        [PIPE_B_EVEN_FIELD] = "Pipe B even field",
+        [PIPE_B_VBLANK] = "Pipe B vblank",
+        [PIPE_C_VBLANK] = "Pipe C vblank",
+        [DPST_PHASE_IN] = "DPST phase in event",
+        [DPST_HISTOGRAM] = "DPST histogram event",
+        [GSE] = "GSE",
+        [DP_A_HOTPLUG] = "DP A Hotplug",
+        [AUX_CHANNEL_A] = "AUX Channel A",
+        [PERF_COUNTER] = "Performance counter",
+        [POISON] = "Poison",
+        [GTT_FAULT] = "GTT fault",
+        [PRIMARY_A_FLIP_DONE] = "Primary Plane A flip done",
+        [PRIMARY_B_FLIP_DONE] = "Primary Plane B flip done",
+        [PRIMARY_C_FLIP_DONE] = "Primary Plane C flip done",
+        [SPRITE_A_FLIP_DONE] = "Sprite Plane A flip done",
+        [SPRITE_B_FLIP_DONE] = "Sprite Plane B flip done",
+        [SPRITE_C_FLIP_DONE] = "Sprite Plane C flip done",
+
+        [PCU_THERMAL] = "PCU Thermal Event",
+        [PCU_PCODE2DRIVER_MAILBOX] = "PCU pcode2driver mailbox event",
+
+        [FDI_RX_INTERRUPTS_TRANSCODER_A] = "FDI RX Interrupts Combined A",
+        [AUDIO_CP_CHANGE_TRANSCODER_A] = "Audio CP Change Transcoder A",
+        [AUDIO_CP_REQUEST_TRANSCODER_A] = "Audio CP Request Transcoder A",
+        [FDI_RX_INTERRUPTS_TRANSCODER_B] = "FDI RX Interrupts Combined B",
+        [AUDIO_CP_CHANGE_TRANSCODER_B] = "Audio CP Change Transcoder B",
+        [AUDIO_CP_REQUEST_TRANSCODER_B] = "Audio CP Request Transcoder B",
+        [FDI_RX_INTERRUPTS_TRANSCODER_C] = "FDI RX Interrupts Combined C",
+        [AUDIO_CP_CHANGE_TRANSCODER_C] = "Audio CP Change Transcoder C",
+        [AUDIO_CP_REQUEST_TRANSCODER_C] = "Audio CP Request Transcoder C",
+        [ERR_AND_DBG] = "South Error and Debug Interupts Combined",
+        [GMBUS] = "Gmbus",
+        [SDVO_B_HOTPLUG] = "SDVO B hotplug",
+        [CRT_HOTPLUG] = "CRT Hotplug",
+        [DP_B_HOTPLUG] = "DisplayPort/HDMI/DVI B Hotplug",
+        [DP_C_HOTPLUG] = "DisplayPort/HDMI/DVI C Hotplug",
+        [DP_D_HOTPLUG] = "DisplayPort/HDMI/DVI D Hotplug",
+        [AUX_CHANNEL_B] = "AUX Channel B",
+        [AUX_CHANNEL_C] = "AUX Channel C",
+        [AUX_CHANNEL_D] = "AUX Channel D",
+        [AUDIO_POWER_STATE_CHANGE_B] = "Audio Power State change Port B",
+        [AUDIO_POWER_STATE_CHANGE_C] = "Audio Power State change Port C",
+        [AUDIO_POWER_STATE_CHANGE_D] = "Audio Power State change Port D",
+
+        [INTEL_GVT_EVENT_RESERVED] = "RESERVED EVENTS!!!",
+};
+
+static inline struct intel_gvt_irq_info *regbase_to_irq_info(
+                struct intel_gvt *gvt,
+                unsigned int reg)
+{
+        struct intel_gvt_irq *irq = &gvt->irq;
+        int i;
+
+        for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) {
+                if (i915_mmio_reg_offset(irq->info[i]->reg_base) == reg)
+                        return irq->info[i];
+        }
+
+        return NULL;
+}
+
+/**
+ * intel_vgpu_reg_imr_handler - Generic IMR register emulation write handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the generic IMR register bit change
+ * behavior.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu,
+        unsigned int reg, void *p_data, unsigned int bytes)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+        u32 changed, masked, unmasked;
+        u32 imr = *(u32 *)p_data;
+
+        gvt_dbg_irq("write IMR %x with val %x\n",
+                reg, imr);
+
+        gvt_dbg_irq("old vIMR %x\n", vgpu_vreg(vgpu, reg));
+
+        /* figure out newly masked/unmasked bits */
+        changed = vgpu_vreg(vgpu, reg) ^ imr;
+        masked = (vgpu_vreg(vgpu, reg) & changed) ^ changed;
+        unmasked = masked ^ changed;
+
+        gvt_dbg_irq("changed %x, masked %x, unmasked %x\n",
+                changed, masked, unmasked);
+
+        vgpu_vreg(vgpu, reg) = imr;
+
+        ops->check_pending_irq(vgpu);
+        gvt_dbg_irq("IRQ: new vIMR %x\n", vgpu_vreg(vgpu, reg));
+        return 0;
+}
+
+/**
+ * intel_vgpu_reg_master_irq_handler - master IRQ write emulation handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the master IRQ register on gen8+.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu,
+        unsigned int reg, void *p_data, unsigned int bytes)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+        u32 changed, enabled, disabled;
+        u32 ier = *(u32 *)p_data;
+        u32 virtual_ier = vgpu_vreg(vgpu, reg);
+
+        gvt_dbg_irq("write master irq reg %x with val %x\n",
+                reg, ier);
+
+        gvt_dbg_irq("old vreg %x\n", vgpu_vreg(vgpu, reg));
+
+        /*
+         * GEN8_MASTER_IRQ is a special irq register,
+         * only bit 31 is allowed to be modified
+         * and treated as an IER bit.
+         */
+        ier &= GEN8_MASTER_IRQ_CONTROL;
+        virtual_ier &= GEN8_MASTER_IRQ_CONTROL;
+        vgpu_vreg(vgpu, reg) &= ~GEN8_MASTER_IRQ_CONTROL;
+        vgpu_vreg(vgpu, reg) |= ier;
+
+        /* figure out newly enabled/disable bits */
+        changed = virtual_ier ^ ier;
+        enabled = (virtual_ier & changed) ^ changed;
+        disabled = enabled ^ changed;
+
+        gvt_dbg_irq("changed %x, enabled %x, disabled %x\n",
+                        changed, enabled, disabled);
+
+        ops->check_pending_irq(vgpu);
+        gvt_dbg_irq("new vreg %x\n", vgpu_vreg(vgpu, reg));
+        return 0;
+}
+
+/**
+ * intel_vgpu_reg_ier_handler - Generic IER write emulation handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the generic IER register behavior.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu,
+        unsigned int reg, void *p_data, unsigned int bytes)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+        struct intel_gvt_irq_info *info;
+        u32 changed, enabled, disabled;
+        u32 ier = *(u32 *)p_data;
+
+        gvt_dbg_irq("write IER %x with val %x\n",
+                reg, ier);
+
+        gvt_dbg_irq("old vIER %x\n", vgpu_vreg(vgpu, reg));
+
+        /* figure out newly enabled/disable bits */
+        changed = vgpu_vreg(vgpu, reg) ^ ier;
+        enabled = (vgpu_vreg(vgpu, reg) & changed) ^ changed;
+        disabled = enabled ^ changed;
+
+        gvt_dbg_irq("changed %x, enabled %x, disabled %x\n",
+                        changed, enabled, disabled);
+        vgpu_vreg(vgpu, reg) = ier;
+
+        info = regbase_to_irq_info(gvt, ier_to_regbase(reg));
+        if (WARN_ON(!info))
+                return -EINVAL;
+
+        if (info->has_upstream_irq)
+                update_upstream_irq(vgpu, info);
+
+        ops->check_pending_irq(vgpu);
+        gvt_dbg_irq("new vIER %x\n", vgpu_vreg(vgpu, reg));
+        return 0;
+}
+
+/**
+ * intel_vgpu_reg_iir_handler - Generic IIR write emulation handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the generic IIR register behavior.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg,
+        void *p_data, unsigned int bytes)
+{
+        struct intel_gvt_irq_info *info = regbase_to_irq_info(vgpu->gvt,
+                iir_to_regbase(reg));
+        u32 iir = *(u32 *)p_data;
+
+        gvt_dbg_irq("write IIR %x with val %x\n", reg, iir);
+
+        if (WARN_ON(!info))
+                return -EINVAL;
+
+        vgpu_vreg(vgpu, reg) &= ~iir;
+
+        if (info->has_upstream_irq)
+                update_upstream_irq(vgpu, info);
+        return 0;
+}
+
+static struct intel_gvt_irq_map gen8_irq_map[] = {
+        { INTEL_GVT_IRQ_INFO_MASTER, 0, INTEL_GVT_IRQ_INFO_GT0, 0xffff },
+        { INTEL_GVT_IRQ_INFO_MASTER, 1, INTEL_GVT_IRQ_INFO_GT0, 0xffff0000 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 2, INTEL_GVT_IRQ_INFO_GT1, 0xffff },
+        { INTEL_GVT_IRQ_INFO_MASTER, 3, INTEL_GVT_IRQ_INFO_GT1, 0xffff0000 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 4, INTEL_GVT_IRQ_INFO_GT2, 0xffff },
+        { INTEL_GVT_IRQ_INFO_MASTER, 6, INTEL_GVT_IRQ_INFO_GT3, 0xffff },
+        { INTEL_GVT_IRQ_INFO_MASTER, 16, INTEL_GVT_IRQ_INFO_DE_PIPE_A, ~0 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 17, INTEL_GVT_IRQ_INFO_DE_PIPE_B, ~0 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 18, INTEL_GVT_IRQ_INFO_DE_PIPE_C, ~0 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 20, INTEL_GVT_IRQ_INFO_DE_PORT, ~0 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 22, INTEL_GVT_IRQ_INFO_DE_MISC, ~0 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 23, INTEL_GVT_IRQ_INFO_PCH, ~0 },
+        { INTEL_GVT_IRQ_INFO_MASTER, 30, INTEL_GVT_IRQ_INFO_PCU, ~0 },
+        { -1, -1, ~0 },
+};
+
+static void update_upstream_irq(struct intel_vgpu *vgpu,
+                struct intel_gvt_irq_info *info)
+{
+        struct intel_gvt_irq *irq = &vgpu->gvt->irq;
+        struct intel_gvt_irq_map *map = irq->irq_map;
+        struct intel_gvt_irq_info *up_irq_info = NULL;
+        u32 set_bits = 0;
+        u32 clear_bits = 0;
+        int bit;
+        u32 val = vgpu_vreg(vgpu,
+                        regbase_to_iir(i915_mmio_reg_offset(info->reg_base)))
+                & vgpu_vreg(vgpu,
+                        regbase_to_ier(i915_mmio_reg_offset(info->reg_base)));
+
+        if (!info->has_upstream_irq)
+                return;
+
+        for (map = irq->irq_map; map->up_irq_bit != -1; map++) {
+                if (info->group != map->down_irq_group)
+                        continue;
+
+                if (!up_irq_info)
+                        up_irq_info = irq->info[map->up_irq_group];
+                else
+                        WARN_ON(up_irq_info != irq->info[map->up_irq_group]);
+
+                bit = map->up_irq_bit;
+
+                if (val & map->down_irq_bitmask)
+                        set_bits |= (1 << bit);
+                else
+                        clear_bits |= (1 << bit);
+        }
+
+        WARN_ON(!up_irq_info);
+
+        if (up_irq_info->group == INTEL_GVT_IRQ_INFO_MASTER) {
+                u32 isr = i915_mmio_reg_offset(up_irq_info->reg_base);
+
+                vgpu_vreg(vgpu, isr) &= ~clear_bits;
+                vgpu_vreg(vgpu, isr) |= set_bits;
+        } else {
+                u32 iir = regbase_to_iir(
+                        i915_mmio_reg_offset(up_irq_info->reg_base));
+                u32 imr = regbase_to_imr(
+                        i915_mmio_reg_offset(up_irq_info->reg_base));
+
+                vgpu_vreg(vgpu, iir) |= (set_bits & ~vgpu_vreg(vgpu, imr));
+        }
+
+        if (up_irq_info->has_upstream_irq)
+                update_upstream_irq(vgpu, up_irq_info);
+}
+
+static void init_irq_map(struct intel_gvt_irq *irq)
+{
+        struct intel_gvt_irq_map *map;
+        struct intel_gvt_irq_info *up_info, *down_info;
+        int up_bit;
+
+        for (map = irq->irq_map; map->up_irq_bit != -1; map++) {
+                up_info = irq->info[map->up_irq_group];
+                up_bit = map->up_irq_bit;
+                down_info = irq->info[map->down_irq_group];
+
+                set_bit(up_bit, up_info->downstream_irq_bitmap);
+                down_info->has_upstream_irq = true;
+
+                gvt_dbg_irq("[up] grp %d bit %d -> [down] grp %d bitmask %x\n",
+                        up_info->group, up_bit,
+                        down_info->group, map->down_irq_bitmask);
+        }
+}
+
+/* =======================vEvent injection===================== */
+static int inject_virtual_interrupt(struct intel_vgpu *vgpu)
+{
+        return intel_gvt_hypervisor_inject_msi(vgpu);
+}
+
+static void propagate_event(struct intel_gvt_irq *irq,
+        enum intel_gvt_event_type event, struct intel_vgpu *vgpu)
+{
+        struct intel_gvt_irq_info *info;
+        unsigned int reg_base;
+        int bit;
+
+        info = get_irq_info(irq, event);
+        if (WARN_ON(!info))
+                return;
+
+        reg_base = i915_mmio_reg_offset(info->reg_base);
+        bit = irq->events[event].bit;
+
+        if (!test_bit(bit, (void *)&vgpu_vreg(vgpu,
+                                        regbase_to_imr(reg_base)))) {
+                gvt_dbg_irq("set bit (%d) for (%s) for vgpu (%d)\n",
+                                bit, irq_name[event], vgpu->id);
+                set_bit(bit, (void *)&vgpu_vreg(vgpu,
+                                        regbase_to_iir(reg_base)));
+        }
+}
+
+/* =======================vEvent Handlers===================== */
+static void handle_default_event_virt(struct intel_gvt_irq *irq,
+        enum intel_gvt_event_type event, struct intel_vgpu *vgpu)
+{
+        if (!vgpu->irq.irq_warn_once[event]) {
+                gvt_dbg_core("vgpu%d: IRQ receive event %d (%s)\n",
+                        vgpu->id, event, irq_name[event]);
+                vgpu->irq.irq_warn_once[event] = true;
+        }
+        propagate_event(irq, event, vgpu);
+}
+
+/* =====================GEN specific logic======================= */
+/* GEN8 interrupt routines. */
+
+#define DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(regname, regbase) \
+static struct intel_gvt_irq_info gen8_##regname##_info = { \
+        .name = #regname"-IRQ", \
+        .reg_base = (regbase), \
+        .bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] = \
+                INTEL_GVT_EVENT_RESERVED}, \
+}
+
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt0, GEN8_GT_ISR(0));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt1, GEN8_GT_ISR(1));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt2, GEN8_GT_ISR(2));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt3, GEN8_GT_ISR(3));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_a, GEN8_DE_PIPE_ISR(PIPE_A));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_b, GEN8_DE_PIPE_ISR(PIPE_B));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_c, GEN8_DE_PIPE_ISR(PIPE_C));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_port, GEN8_DE_PORT_ISR);
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_misc, GEN8_DE_MISC_ISR);
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(pcu, GEN8_PCU_ISR);
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(master, GEN8_MASTER_IRQ);
+
+static struct intel_gvt_irq_info gvt_base_pch_info = {
+        .name = "PCH-IRQ",
+        .reg_base = SDEISR,
+        .bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] =
+                INTEL_GVT_EVENT_RESERVED},
+};
+
+static void gen8_check_pending_irq(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt_irq *irq = &vgpu->gvt->irq;
+        int i;
+
+        if (!(vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ)) &
+                                GEN8_MASTER_IRQ_CONTROL))
+                return;
+
+        for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) {
+                struct intel_gvt_irq_info *info = irq->info[i];
+                u32 reg_base;
+
+                if (!info->has_upstream_irq)
+                        continue;
+
+                reg_base = i915_mmio_reg_offset(info->reg_base);
+                if ((vgpu_vreg(vgpu, regbase_to_iir(reg_base))
+                                & vgpu_vreg(vgpu, regbase_to_ier(reg_base))))
+                        update_upstream_irq(vgpu, info);
+        }
+
+        if (vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ))
+                        & ~GEN8_MASTER_IRQ_CONTROL)
+                inject_virtual_interrupt(vgpu);
+}
+
+static void gen8_init_irq(
+                struct intel_gvt_irq *irq)
+{
+        struct intel_gvt *gvt = irq_to_gvt(irq);
+
+#define SET_BIT_INFO(s, b, e, i)                \
+        do {                                    \
+                s->events[e].bit = b;           \
+                s->events[e].info = s->info[i]; \
+                s->info[i]->bit_to_event[b] = e;\
+        } while (0)
+
+#define SET_IRQ_GROUP(s, g, i) \
+        do { \
+                s->info[g] = i; \
+                (i)->group = g; \
+                set_bit(g, s->irq_info_bitmap); \
+        } while (0)
+
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_MASTER, &gen8_master_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT0, &gen8_gt0_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT1, &gen8_gt1_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT2, &gen8_gt2_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT3, &gen8_gt3_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_A, &gen8_de_pipe_a_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_B, &gen8_de_pipe_b_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_C, &gen8_de_pipe_c_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PORT, &gen8_de_port_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_MISC, &gen8_de_misc_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCU, &gen8_pcu_info);
+        SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCH, &gvt_base_pch_info);
+
+        /* GEN8 level 2 interrupts. */
+
+        /* GEN8 interrupt GT0 events */
+        SET_BIT_INFO(irq, 0, RCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0);
+        SET_BIT_INFO(irq, 4, RCS_PIPE_CONTROL, INTEL_GVT_IRQ_INFO_GT0);
+        SET_BIT_INFO(irq, 8, RCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0);
+
+        SET_BIT_INFO(irq, 16, BCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0);
+        SET_BIT_INFO(irq, 20, BCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT0);
+        SET_BIT_INFO(irq, 24, BCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0);
+
+        /* GEN8 interrupt GT1 events */
+        SET_BIT_INFO(irq, 0, VCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT1);
+        SET_BIT_INFO(irq, 4, VCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT1);
+        SET_BIT_INFO(irq, 8, VCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT1);
+
+        if (HAS_BSD2(gvt->dev_priv)) {
+                SET_BIT_INFO(irq, 16, VCS2_MI_USER_INTERRUPT,
+                        INTEL_GVT_IRQ_INFO_GT1);
+                SET_BIT_INFO(irq, 20, VCS2_MI_FLUSH_DW,
+                        INTEL_GVT_IRQ_INFO_GT1);
+                SET_BIT_INFO(irq, 24, VCS2_AS_CONTEXT_SWITCH,
+                        INTEL_GVT_IRQ_INFO_GT1);
+        }
+
+        /* GEN8 interrupt GT3 events */
+        SET_BIT_INFO(irq, 0, VECS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT3);
+        SET_BIT_INFO(irq, 4, VECS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT3);
+        SET_BIT_INFO(irq, 8, VECS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT3);
+
+        SET_BIT_INFO(irq, 0, PIPE_A_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+        SET_BIT_INFO(irq, 0, PIPE_B_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+        SET_BIT_INFO(irq, 0, PIPE_C_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+
+        /* GEN8 interrupt DE PORT events */
+        SET_BIT_INFO(irq, 0, AUX_CHANNEL_A, INTEL_GVT_IRQ_INFO_DE_PORT);
+        SET_BIT_INFO(irq, 3, DP_A_HOTPLUG, INTEL_GVT_IRQ_INFO_DE_PORT);
+
+        /* GEN8 interrupt DE MISC events */
+        SET_BIT_INFO(irq, 0, GSE, INTEL_GVT_IRQ_INFO_DE_MISC);
+
+        /* PCH events */
+        SET_BIT_INFO(irq, 17, GMBUS, INTEL_GVT_IRQ_INFO_PCH);
+        SET_BIT_INFO(irq, 19, CRT_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+        SET_BIT_INFO(irq, 21, DP_B_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+        SET_BIT_INFO(irq, 22, DP_C_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+        SET_BIT_INFO(irq, 23, DP_D_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+
+        if (IS_BROADWELL(gvt->dev_priv)) {
+                SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_PCH);
+                SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_PCH);
+                SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_PCH);
+
+                SET_BIT_INFO(irq, 4, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+                SET_BIT_INFO(irq, 5, SPRITE_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+
+                SET_BIT_INFO(irq, 4, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+                SET_BIT_INFO(irq, 5, SPRITE_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+
+                SET_BIT_INFO(irq, 4, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+                SET_BIT_INFO(irq, 5, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+        } else if (IS_SKYLAKE(gvt->dev_priv)) {
+                SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_DE_PORT);
+                SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_DE_PORT);
+                SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_DE_PORT);
+
+                SET_BIT_INFO(irq, 3, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+                SET_BIT_INFO(irq, 3, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+                SET_BIT_INFO(irq, 3, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+        }
+
+        /* GEN8 interrupt PCU events */
+        SET_BIT_INFO(irq, 24, PCU_THERMAL, INTEL_GVT_IRQ_INFO_PCU);
+        SET_BIT_INFO(irq, 25, PCU_PCODE2DRIVER_MAILBOX, INTEL_GVT_IRQ_INFO_PCU);
+}
+
+static struct intel_gvt_irq_ops gen8_irq_ops = {
+        .init_irq = gen8_init_irq,
+        .check_pending_irq = gen8_check_pending_irq,
+};
+
+/**
+ * intel_vgpu_trigger_virtual_event - Trigger a virtual event for a vGPU
+ * @vgpu: a vGPU
+ * @event: interrupt event
+ *
+ * This function is used to trigger a virtual interrupt event for vGPU.
+ * The caller provides the event to be triggered, the framework itself
+ * will emulate the IRQ register bit change.
+ *
+ */
+void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu,
+        enum intel_gvt_event_type event)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_irq *irq = &gvt->irq;
+        gvt_event_virt_handler_t handler;
+        struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+
+        handler = get_event_virt_handler(irq, event);
+        WARN_ON(!handler);
+
+        handler(irq, event, vgpu);
+
+        ops->check_pending_irq(vgpu);
+}
+
+static void init_events(
+        struct intel_gvt_irq *irq)
+{
+        int i;
+
+        for (i = 0; i < INTEL_GVT_EVENT_MAX; i++) {
+                irq->events[i].info = NULL;
+                irq->events[i].v_handler = handle_default_event_virt;
+        }
+}
+
+static enum hrtimer_restart vblank_timer_fn(struct hrtimer *data)
+{
+        struct intel_gvt_vblank_timer *vblank_timer;
+        struct intel_gvt_irq *irq;
+        struct intel_gvt *gvt;
+
+        vblank_timer = container_of(data, struct intel_gvt_vblank_timer, timer);
+        irq = container_of(vblank_timer, struct intel_gvt_irq, vblank_timer);
+        gvt = container_of(irq, struct intel_gvt, irq);
+
+        intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EMULATE_VBLANK);
+        hrtimer_add_expires_ns(&vblank_timer->timer, vblank_timer->period);
+        return HRTIMER_RESTART;
+}
+
+/**
+ * intel_gvt_clean_irq - clean up GVT-g IRQ emulation subsystem
+ * @gvt: a GVT device
+ *
+ * This function is called at driver unloading stage, to clean up GVT-g IRQ
+ * emulation subsystem.
+ *
+ */
+void intel_gvt_clean_irq(struct intel_gvt *gvt)
+{
+        struct intel_gvt_irq *irq = &gvt->irq;
+
+        hrtimer_cancel(&irq->vblank_timer.timer);
+}
+
+#define VBLNAK_TIMER_PERIOD 16000000
+
+/**
+ * intel_gvt_init_irq - initialize GVT-g IRQ emulation subsystem
+ * @gvt: a GVT device
+ *
+ * This function is called at driver loading stage, to initialize the GVT-g IRQ
+ * emulation subsystem.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_init_irq(struct intel_gvt *gvt)
+{
+        struct intel_gvt_irq *irq = &gvt->irq;
+        struct intel_gvt_vblank_timer *vblank_timer = &irq->vblank_timer;
+
+        gvt_dbg_core("init irq framework\n");
+
+        if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+                irq->ops = &gen8_irq_ops;
+                irq->irq_map = gen8_irq_map;
+        } else {
+                WARN_ON(1);
+                return -ENODEV;
+        }
+
+        /* common event initialization */
+        init_events(irq);
+
+        /* gen specific initialization */
+        irq->ops->init_irq(irq);
+
+        init_irq_map(irq);
+
+        hrtimer_init(&vblank_timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+        vblank_timer->timer.function = vblank_timer_fn;
+        vblank_timer->period = VBLNAK_TIMER_PERIOD;
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/interrupt.h b/drivers/gpu/drm/i915/gvt/interrupt.h
new file mode 100644
index 000000000000..5313fb1b33e1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/interrupt.h
@@ -0,0 +1,233 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min he <min.he@intel.com>
+ *
+ */
+
+#ifndef _GVT_INTERRUPT_H_
+#define _GVT_INTERRUPT_H_
+
+enum intel_gvt_event_type {
+        RCS_MI_USER_INTERRUPT = 0,
+        RCS_DEBUG,
+        RCS_MMIO_SYNC_FLUSH,
+        RCS_CMD_STREAMER_ERR,
+        RCS_PIPE_CONTROL,
+        RCS_L3_PARITY_ERR,
+        RCS_WATCHDOG_EXCEEDED,
+        RCS_PAGE_DIRECTORY_FAULT,
+        RCS_AS_CONTEXT_SWITCH,
+        RCS_MONITOR_BUFF_HALF_FULL,
+
+        VCS_MI_USER_INTERRUPT,
+        VCS_MMIO_SYNC_FLUSH,
+        VCS_CMD_STREAMER_ERR,
+        VCS_MI_FLUSH_DW,
+        VCS_WATCHDOG_EXCEEDED,
+        VCS_PAGE_DIRECTORY_FAULT,
+        VCS_AS_CONTEXT_SWITCH,
+
+        VCS2_MI_USER_INTERRUPT,
+        VCS2_MI_FLUSH_DW,
+        VCS2_AS_CONTEXT_SWITCH,
+
+        BCS_MI_USER_INTERRUPT,
+        BCS_MMIO_SYNC_FLUSH,
+        BCS_CMD_STREAMER_ERR,
+        BCS_MI_FLUSH_DW,
+        BCS_PAGE_DIRECTORY_FAULT,
+        BCS_AS_CONTEXT_SWITCH,
+
+        VECS_MI_USER_INTERRUPT,
+        VECS_MI_FLUSH_DW,
+        VECS_AS_CONTEXT_SWITCH,
+
+        PIPE_A_FIFO_UNDERRUN,
+        PIPE_B_FIFO_UNDERRUN,
+        PIPE_A_CRC_ERR,
+        PIPE_B_CRC_ERR,
+        PIPE_A_CRC_DONE,
+        PIPE_B_CRC_DONE,
+        PIPE_A_ODD_FIELD,
+        PIPE_B_ODD_FIELD,
+        PIPE_A_EVEN_FIELD,
+        PIPE_B_EVEN_FIELD,
+        PIPE_A_LINE_COMPARE,
+        PIPE_B_LINE_COMPARE,
+        PIPE_C_LINE_COMPARE,
+        PIPE_A_VBLANK,
+        PIPE_B_VBLANK,
+        PIPE_C_VBLANK,
+        PIPE_A_VSYNC,
+        PIPE_B_VSYNC,
+        PIPE_C_VSYNC,
+        PRIMARY_A_FLIP_DONE,
+        PRIMARY_B_FLIP_DONE,
+        PRIMARY_C_FLIP_DONE,
+        SPRITE_A_FLIP_DONE,
+        SPRITE_B_FLIP_DONE,
+        SPRITE_C_FLIP_DONE,
+
+        PCU_THERMAL,
+        PCU_PCODE2DRIVER_MAILBOX,
+
+        DPST_PHASE_IN,
+        DPST_HISTOGRAM,
+        GSE,
+        DP_A_HOTPLUG,
+        AUX_CHANNEL_A,
+        PERF_COUNTER,
+        POISON,
+        GTT_FAULT,
+        ERROR_INTERRUPT_COMBINED,
+
+        FDI_RX_INTERRUPTS_TRANSCODER_A,
+        AUDIO_CP_CHANGE_TRANSCODER_A,
+        AUDIO_CP_REQUEST_TRANSCODER_A,
+        FDI_RX_INTERRUPTS_TRANSCODER_B,
+        AUDIO_CP_CHANGE_TRANSCODER_B,
+        AUDIO_CP_REQUEST_TRANSCODER_B,
+        FDI_RX_INTERRUPTS_TRANSCODER_C,
+        AUDIO_CP_CHANGE_TRANSCODER_C,
+        AUDIO_CP_REQUEST_TRANSCODER_C,
+        ERR_AND_DBG,
+        GMBUS,
+        SDVO_B_HOTPLUG,
+        CRT_HOTPLUG,
+        DP_B_HOTPLUG,
+        DP_C_HOTPLUG,
+        DP_D_HOTPLUG,
+        AUX_CHANNEL_B,
+        AUX_CHANNEL_C,
+        AUX_CHANNEL_D,
+        AUDIO_POWER_STATE_CHANGE_B,
+        AUDIO_POWER_STATE_CHANGE_C,
+        AUDIO_POWER_STATE_CHANGE_D,
+
+        INTEL_GVT_EVENT_RESERVED,
+        INTEL_GVT_EVENT_MAX,
+};
+
+struct intel_gvt_irq;
+struct intel_gvt;
+
+typedef void (*gvt_event_virt_handler_t)(struct intel_gvt_irq *irq,
+        enum intel_gvt_event_type event, struct intel_vgpu *vgpu);
+
+struct intel_gvt_irq_ops {
+        void (*init_irq)(struct intel_gvt_irq *irq);
+        void (*check_pending_irq)(struct intel_vgpu *vgpu);
+};
+
+/* the list of physical interrupt control register groups */
+enum intel_gvt_irq_type {
+        INTEL_GVT_IRQ_INFO_GT,
+        INTEL_GVT_IRQ_INFO_DPY,
+        INTEL_GVT_IRQ_INFO_PCH,
+        INTEL_GVT_IRQ_INFO_PM,
+
+        INTEL_GVT_IRQ_INFO_MASTER,
+        INTEL_GVT_IRQ_INFO_GT0,
+        INTEL_GVT_IRQ_INFO_GT1,
+        INTEL_GVT_IRQ_INFO_GT2,
+        INTEL_GVT_IRQ_INFO_GT3,
+        INTEL_GVT_IRQ_INFO_DE_PIPE_A,
+        INTEL_GVT_IRQ_INFO_DE_PIPE_B,
+        INTEL_GVT_IRQ_INFO_DE_PIPE_C,
+        INTEL_GVT_IRQ_INFO_DE_PORT,
+        INTEL_GVT_IRQ_INFO_DE_MISC,
+        INTEL_GVT_IRQ_INFO_AUD,
+        INTEL_GVT_IRQ_INFO_PCU,
+
+        INTEL_GVT_IRQ_INFO_MAX,
+};
+
+#define INTEL_GVT_IRQ_BITWIDTH  32
+
+/* device specific interrupt bit definitions */
+struct intel_gvt_irq_info {
+        char *name;
+        i915_reg_t reg_base;
+        enum intel_gvt_event_type bit_to_event[INTEL_GVT_IRQ_BITWIDTH];
+        unsigned long warned;
+        int group;
+        DECLARE_BITMAP(downstream_irq_bitmap, INTEL_GVT_IRQ_BITWIDTH);
+        bool has_upstream_irq;
+};
+
+/* per-event information */
+struct intel_gvt_event_info {
+        int bit;                                /* map to register bit */
+        int policy;                             /* forwarding policy */
+        struct intel_gvt_irq_info *info;        /* register info */
+        gvt_event_virt_handler_t v_handler;     /* for v_event */
+};
+
+struct intel_gvt_irq_map {
+        int up_irq_group;
+        int up_irq_bit;
+        int down_irq_group;
+        u32 down_irq_bitmask;
+};
+
+struct intel_gvt_vblank_timer {
+        struct hrtimer timer;
+        u64 period;
+};
+
+/* structure containing device specific IRQ state */
+struct intel_gvt_irq {
+        struct intel_gvt_irq_ops *ops;
+        struct intel_gvt_irq_info *info[INTEL_GVT_IRQ_INFO_MAX];
+        DECLARE_BITMAP(irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX);
+        struct intel_gvt_event_info events[INTEL_GVT_EVENT_MAX];
+        DECLARE_BITMAP(pending_events, INTEL_GVT_EVENT_MAX);
+        struct intel_gvt_irq_map *irq_map;
+        struct intel_gvt_vblank_timer vblank_timer;
+};
+
+int intel_gvt_init_irq(struct intel_gvt *gvt);
+void intel_gvt_clean_irq(struct intel_gvt *gvt);
+
+void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu,
+        enum intel_gvt_event_type event);
+
+int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg,
+        void *p_data, unsigned int bytes);
+int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu,
+        unsigned int reg, void *p_data, unsigned int bytes);
+int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu,
+        unsigned int reg, void *p_data, unsigned int bytes);
+int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu,
+        unsigned int reg, void *p_data, unsigned int bytes);
+
+int gvt_ring_id_to_pipe_control_notify_event(int ring_id);
+int gvt_ring_id_to_mi_flush_dw_event(int ring_id);
+int gvt_ring_id_to_mi_user_interrupt_event(int ring_id);
+
+#endif /* _GVT_INTERRUPT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
new file mode 100644
index 000000000000..585b01f63254
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -0,0 +1,306 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Dexuan Cui
+ *
+ * Contributors:
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+/**
+ * intel_vgpu_gpa_to_mmio_offset - translate a GPA to MMIO offset
+ * @vgpu: a vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
+{
+        u64 gttmmio_gpa = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0) &
+                          ~GENMASK(3, 0);
+        return gpa - gttmmio_gpa;
+}
+
+#define reg_is_mmio(gvt, reg)  \
+        (reg >= 0 && reg < gvt->device_info.mmio_size)
+
+#define reg_is_gtt(gvt, reg)   \
+        (reg >= gvt->device_info.gtt_start_offset \
+         && reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt))
+
+/**
+ * intel_vgpu_emulate_mmio_read - emulate MMIO read
+ * @vgpu: a vGPU
+ * @pa: guest physical address
+ * @p_data: data return buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_emulate_mmio_read(void *__vgpu, uint64_t pa,
+                void *p_data, unsigned int bytes)
+{
+        struct intel_vgpu *vgpu = __vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_mmio_info *mmio;
+        unsigned int offset = 0;
+        int ret = -EINVAL;
+
+        mutex_lock(&gvt->lock);
+
+        if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
+                struct intel_vgpu_guest_page *gp;
+
+                gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
+                if (gp) {
+                        ret = intel_gvt_hypervisor_read_gpa(vgpu, pa,
+                                        p_data, bytes);
+                        if (ret) {
+                                gvt_err("vgpu%d: guest page read error %d, "
+                                        "gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n",
+                                        vgpu->id, ret,
+                                        gp->gfn, pa, *(u32 *)p_data, bytes);
+                        }
+                        mutex_unlock(&gvt->lock);
+                        return ret;
+                }
+        }
+
+        offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa);
+
+        if (WARN_ON(bytes > 8))
+                goto err;
+
+        if (reg_is_gtt(gvt, offset)) {
+                if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8)))
+                        goto err;
+                if (WARN_ON(bytes != 4 && bytes != 8))
+                        goto err;
+                if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1)))
+                        goto err;
+
+                ret = intel_vgpu_emulate_gtt_mmio_read(vgpu, offset,
+                                p_data, bytes);
+                if (ret)
+                        goto err;
+                mutex_unlock(&gvt->lock);
+                return ret;
+        }
+
+        if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) {
+                ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, p_data, bytes);
+                mutex_unlock(&gvt->lock);
+                return ret;
+        }
+
+        if (WARN_ON(!reg_is_mmio(gvt, offset + bytes - 1)))
+                goto err;
+
+        mmio = intel_gvt_find_mmio_info(gvt, rounddown(offset, 4));
+        if (!mmio && !vgpu->mmio.disable_warn_untrack) {
+                gvt_err("vgpu%d: read untracked MMIO %x len %d val %x\n",
+                                vgpu->id, offset, bytes, *(u32 *)p_data);
+
+                if (offset == 0x206c) {
+                        gvt_err("------------------------------------------\n");
+                        gvt_err("vgpu%d: likely triggers a gfx reset\n",
+                        vgpu->id);
+                        gvt_err("------------------------------------------\n");
+                        vgpu->mmio.disable_warn_untrack = true;
+                }
+        }
+
+        if (!intel_gvt_mmio_is_unalign(gvt, offset)) {
+                if (WARN_ON(!IS_ALIGNED(offset, bytes)))
+                        goto err;
+        }
+
+        if (mmio) {
+                if (!intel_gvt_mmio_is_unalign(gvt, mmio->offset)) {
+                        if (WARN_ON(offset + bytes > mmio->offset + mmio->size))
+                                goto err;
+                        if (WARN_ON(mmio->offset != offset))
+                                goto err;
+                }
+                ret = mmio->read(vgpu, offset, p_data, bytes);
+        } else
+                ret = intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
+
+        if (ret)
+                goto err;
+
+        intel_gvt_mmio_set_accessed(gvt, offset);
+        mutex_unlock(&gvt->lock);
+        return 0;
+err:
+        gvt_err("vgpu%d: fail to emulate MMIO read %08x len %d\n",
+                        vgpu->id, offset, bytes);
+        mutex_unlock(&gvt->lock);
+        return ret;
+}
+
+/**
+ * intel_vgpu_emulate_mmio_write - emulate MMIO write
+ * @vgpu: a vGPU
+ * @pa: guest physical address
+ * @p_data: write data buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_emulate_mmio_write(void *__vgpu, uint64_t pa,
+                void *p_data, unsigned int bytes)
+{
+        struct intel_vgpu *vgpu = __vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_mmio_info *mmio;
+        unsigned int offset = 0;
+        u32 old_vreg = 0, old_sreg = 0;
+        int ret = -EINVAL;
+
+        mutex_lock(&gvt->lock);
+
+        if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
+                struct intel_vgpu_guest_page *gp;
+
+                gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
+                if (gp) {
+                        ret = gp->handler(gp, pa, p_data, bytes);
+                        if (ret) {
+                                gvt_err("vgpu%d: guest page write error %d, "
+                                        "gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n",
+                                        vgpu->id, ret,
+                                        gp->gfn, pa, *(u32 *)p_data, bytes);
+                        }
+                        mutex_unlock(&gvt->lock);
+                        return ret;
+                }
+        }
+
+        offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa);
+
+        if (WARN_ON(bytes > 8))
+                goto err;
+
+        if (reg_is_gtt(gvt, offset)) {
+                if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8)))
+                        goto err;
+                if (WARN_ON(bytes != 4 && bytes != 8))
+                        goto err;
+                if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1)))
+                        goto err;
+
+                ret = intel_vgpu_emulate_gtt_mmio_write(vgpu, offset,
+                                p_data, bytes);
+                if (ret)
+                        goto err;
+                mutex_unlock(&gvt->lock);
+                return ret;
+        }
+
+        if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) {
+                ret = intel_gvt_hypervisor_write_gpa(vgpu, pa, p_data, bytes);
+                mutex_unlock(&gvt->lock);
+                return ret;
+        }
+
+        mmio = intel_gvt_find_mmio_info(gvt, rounddown(offset, 4));
+        if (!mmio && !vgpu->mmio.disable_warn_untrack)
+                gvt_err("vgpu%d: write untracked MMIO %x len %d val %x\n",
+                                vgpu->id, offset, bytes, *(u32 *)p_data);
+
+        if (!intel_gvt_mmio_is_unalign(gvt, offset)) {
+                if (WARN_ON(!IS_ALIGNED(offset, bytes)))
+                        goto err;
+        }
+
+        if (mmio) {
+                u64 ro_mask = mmio->ro_mask;
+
+                if (!intel_gvt_mmio_is_unalign(gvt, mmio->offset)) {
+                        if (WARN_ON(offset + bytes > mmio->offset + mmio->size))
+                                goto err;
+                        if (WARN_ON(mmio->offset != offset))
+                                goto err;
+                }
+
+                if (intel_gvt_mmio_has_mode_mask(gvt, mmio->offset)) {
+                        old_vreg = vgpu_vreg(vgpu, offset);
+                        old_sreg = vgpu_sreg(vgpu, offset);
+                }
+
+                if (!ro_mask) {
+                        ret = mmio->write(vgpu, offset, p_data, bytes);
+                } else {
+                        /* Protect RO bits like HW */
+                        u64 data = 0;
+
+                        /* all register bits are RO. */
+                        if (ro_mask == ~(u64)0) {
+                                gvt_err("vgpu%d: try to write RO reg %x\n",
+                                                vgpu->id, offset);
+                                ret = 0;
+                                goto out;
+                        }
+                        /* keep the RO bits in the virtual register */
+                        memcpy(&data, p_data, bytes);
+                        data &= ~mmio->ro_mask;
+                        data |= vgpu_vreg(vgpu, offset) & mmio->ro_mask;
+                        ret = mmio->write(vgpu, offset, &data, bytes);
+                }
+
+                /* higher 16bits of mode ctl regs are mask bits for change */
+                if (intel_gvt_mmio_has_mode_mask(gvt, mmio->offset)) {
+                        u32 mask = vgpu_vreg(vgpu, offset) >> 16;
+
+                        vgpu_vreg(vgpu, offset) = (old_vreg & ~mask)
+                                | (vgpu_vreg(vgpu, offset) & mask);
+                        vgpu_sreg(vgpu, offset) = (old_sreg & ~mask)
+                                | (vgpu_sreg(vgpu, offset) & mask);
+                }
+        } else
+                ret = intel_vgpu_default_mmio_write(vgpu, offset, p_data,
+                                bytes);
+        if (ret)
+                goto err;
+out:
+        intel_gvt_mmio_set_accessed(gvt, offset);
+        mutex_unlock(&gvt->lock);
+        return 0;
+err:
+        gvt_err("vgpu%d: fail to emulate MMIO write %08x len %d\n",
+                        vgpu->id, offset, bytes);
+        mutex_unlock(&gvt->lock);
+        return ret;
+}
diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
new file mode 100644
index 000000000000..9dc739a01892
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/mmio.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Dexuan Cui
+ *
+ * Contributors:
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef _GVT_MMIO_H_
+#define _GVT_MMIO_H_
+
+struct intel_gvt;
+struct intel_vgpu;
+
+#define D_SNB   (1 << 0)
+#define D_IVB   (1 << 1)
+#define D_HSW   (1 << 2)
+#define D_BDW   (1 << 3)
+#define D_SKL   (1 << 4)
+
+#define D_GEN9PLUS      (D_SKL)
+#define D_GEN8PLUS      (D_BDW | D_SKL)
+#define D_GEN75PLUS     (D_HSW | D_BDW | D_SKL)
+#define D_GEN7PLUS      (D_IVB | D_HSW | D_BDW | D_SKL)
+
+#define D_SKL_PLUS      (D_SKL)
+#define D_BDW_PLUS      (D_BDW | D_SKL)
+#define D_HSW_PLUS      (D_HSW | D_BDW | D_SKL)
+#define D_IVB_PLUS      (D_IVB | D_HSW | D_BDW | D_SKL)
+
+#define D_PRE_BDW       (D_SNB | D_IVB | D_HSW)
+#define D_PRE_SKL       (D_SNB | D_IVB | D_HSW | D_BDW)
+#define D_ALL           (D_SNB | D_IVB | D_HSW | D_BDW | D_SKL)
+
+struct intel_gvt_mmio_info {
+        u32 offset;
+        u32 size;
+        u32 length;
+        u32 addr_mask;
+        u64 ro_mask;
+        u32 device;
+        int (*read)(struct intel_vgpu *, unsigned int, void *, unsigned int);
+        int (*write)(struct intel_vgpu *, unsigned int, void *, unsigned int);
+        u32 addr_range;
+        struct hlist_node node;
+};
+
+unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt);
+bool intel_gvt_match_device(struct intel_gvt *gvt, unsigned long device);
+
+int intel_gvt_setup_mmio_info(struct intel_gvt *gvt);
+void intel_gvt_clean_mmio_info(struct intel_gvt *gvt);
+
+struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
+                                                     unsigned int offset);
+#define INTEL_GVT_MMIO_OFFSET(reg) ({ \
+        typeof(reg) __reg = reg; \
+        u32 *offset = (u32 *)&__reg; \
+        *offset; \
+})
+
+int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa);
+int intel_vgpu_emulate_mmio_read(void *__vgpu, u64 pa, void *p_data,
+                                 unsigned int bytes);
+int intel_vgpu_emulate_mmio_write(void *__vgpu, u64 pa, void *p_data,
+                                  unsigned int bytes);
+bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt,
+                                  unsigned int offset);
+bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt, unsigned int offset);
+void intel_gvt_mmio_set_accessed(struct intel_gvt *gvt, unsigned int offset);
+void intel_gvt_mmio_set_cmd_accessed(struct intel_gvt *gvt,
+                                     unsigned int offset);
+bool intel_gvt_mmio_has_mode_mask(struct intel_gvt *gvt, unsigned int offset);
+int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+                                 void *p_data, unsigned int bytes);
+int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+                                  void *p_data, unsigned int bytes);
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index 03601e3ffa7c..67858782d327 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -19,6 +19,15 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Dexuan Cui
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_MPT_H_
@@ -46,4 +55,215 @@ static inline int intel_gvt_hypervisor_detect_host(void)
         return intel_gvt_host.mpt->detect_host();
 }
 
+/**
+ * intel_gvt_hypervisor_attach_vgpu - call hypervisor to initialize vGPU
+ * related stuffs inside hypervisor.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu)
+{
+        return intel_gvt_host.mpt->attach_vgpu(vgpu, &vgpu->handle);
+}
+
+/**
+ * intel_gvt_hypervisor_detach_vgpu - call hypervisor to release vGPU
+ * related stuffs inside hypervisor.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu)
+{
+        intel_gvt_host.mpt->detach_vgpu(vgpu->handle);
+}
+
+#define MSI_CAP_CONTROL(offset) (offset + 2)
+#define MSI_CAP_ADDRESS(offset) (offset + 4)
+#define MSI_CAP_DATA(offset) (offset + 8)
+#define MSI_CAP_EN 0x1
+
+/**
+ * intel_gvt_hypervisor_inject_msi - inject a MSI interrupt into vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_inject_msi(struct intel_vgpu *vgpu)
+{
+        unsigned long offset = vgpu->gvt->device_info.msi_cap_offset;
+        u16 control, data;
+        u32 addr;
+        int ret;
+
+        control = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_CONTROL(offset));
+        addr = *(u32 *)(vgpu_cfg_space(vgpu) + MSI_CAP_ADDRESS(offset));
+        data = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_DATA(offset));
+
+        /* Do not generate MSI if MSIEN is disable */
+        if (!(control & MSI_CAP_EN))
+                return 0;
+
+        if (WARN(control & GENMASK(15, 1), "only support one MSI format\n"))
+                return -EINVAL;
+
+        gvt_dbg_irq("vgpu%d: inject msi address %x data%x\n", vgpu->id, addr,
+                    data);
+
+        ret = intel_gvt_host.mpt->inject_msi(vgpu->handle, addr, data);
+        if (ret)
+                return ret;
+        return 0;
+}
+
+/**
+ * intel_gvt_hypervisor_set_wp_page - translate a host VA into MFN
+ * @p: host kernel virtual address
+ *
+ * Returns:
+ * MFN on success, INTEL_GVT_INVALID_ADDR if failed.
+ */
+static inline unsigned long intel_gvt_hypervisor_virt_to_mfn(void *p)
+{
+        return intel_gvt_host.mpt->from_virt_to_mfn(p);
+}
+
+/**
+ * intel_gvt_hypervisor_set_wp_page - set a guest page to write-protected
+ * @vgpu: a vGPU
+ * @p: intel_vgpu_guest_page
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_set_wp_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *p)
+{
+        int ret;
+
+        if (p->writeprotection)
+                return 0;
+
+        ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, p->gfn);
+        if (ret)
+                return ret;
+        p->writeprotection = true;
+        atomic_inc(&vgpu->gtt.n_write_protected_guest_page);
+        return 0;
+}
+
+/**
+ * intel_gvt_hypervisor_unset_wp_page - remove the write-protection of a
+ * guest page
+ * @vgpu: a vGPU
+ * @p: intel_vgpu_guest_page
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_unset_wp_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *p)
+{
+        int ret;
+
+        if (!p->writeprotection)
+                return 0;
+
+        ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, p->gfn);
+        if (ret)
+                return ret;
+        p->writeprotection = false;
+        atomic_dec(&vgpu->gtt.n_write_protected_guest_page);
+        return 0;
+}
+
+/**
+ * intel_gvt_hypervisor_read_gpa - copy data from GPA to host data buffer
+ * @vgpu: a vGPU
+ * @gpa: guest physical address
+ * @buf: host data buffer
+ * @len: data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_read_gpa(struct intel_vgpu *vgpu,
+                unsigned long gpa, void *buf, unsigned long len)
+{
+        return intel_gvt_host.mpt->read_gpa(vgpu->handle, gpa, buf, len);
+}
+
+/**
+ * intel_gvt_hypervisor_write_gpa - copy data from host data buffer to GPA
+ * @vgpu: a vGPU
+ * @gpa: guest physical address
+ * @buf: host data buffer
+ * @len: data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_write_gpa(struct intel_vgpu *vgpu,
+                unsigned long gpa, void *buf, unsigned long len)
+{
+        return intel_gvt_host.mpt->write_gpa(vgpu->handle, gpa, buf, len);
+}
+
+/**
+ * intel_gvt_hypervisor_gfn_to_mfn - translate a GFN to MFN
+ * @vgpu: a vGPU
+ * @gpfn: guest pfn
+ *
+ * Returns:
+ * MFN on success, INTEL_GVT_INVALID_ADDR if failed.
+ */
+static inline unsigned long intel_gvt_hypervisor_gfn_to_mfn(
+                struct intel_vgpu *vgpu, unsigned long gfn)
+{
+        return intel_gvt_host.mpt->gfn_to_mfn(vgpu->handle, gfn);
+}
+
+enum {
+        GVT_MAP_APERTURE = 0,
+        GVT_MAP_OPREGION,
+};
+
+/**
+ * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN
+ * @vgpu: a vGPU
+ * @gfn: guest PFN
+ * @mfn: host PFN
+ * @nr: amount of PFNs
+ * @map: map or unmap
+ * @type: map type
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_map_gfn_to_mfn(
+                struct intel_vgpu *vgpu, unsigned long gfn,
+                unsigned long mfn, unsigned int nr,
+                bool map, int type)
+{
+        return intel_gvt_host.mpt->map_gfn_to_mfn(vgpu->handle, gfn, mfn, nr,
+                                                  map, type);
+}
+
+/**
+ * intel_gvt_hypervisor_set_trap_area - Trap a guest PA region
+ * @vgpu: a vGPU
+ * @start: the beginning of the guest physical address region
+ * @end: the end of the guest physical address region
+ * @map: map or unmap
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_set_trap_area(
+                struct intel_vgpu *vgpu, u64 start, u64 end, bool map)
+{
+        return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map);
+}
+
 #endif /* _GVT_MPT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c
new file mode 100644
index 000000000000..973c8a9d0b15
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/opregion.c
@@ -0,0 +1,344 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/acpi.h>
+#include "i915_drv.h"
+#include "gvt.h"
+
+static int init_vgpu_opregion(struct intel_vgpu *vgpu, u32 gpa)
+{
+        void __iomem *host_va = vgpu->gvt->opregion.opregion_va;
+        u8 *buf;
+        int i;
+
+        if (WARN((vgpu_opregion(vgpu)->va),
+                        "vgpu%d: opregion has been initialized already.\n",
+                        vgpu->id))
+                return -EINVAL;
+
+        vgpu_opregion(vgpu)->va = (void *)__get_free_pages(GFP_ATOMIC |
+                        GFP_DMA32 | __GFP_ZERO,
+                        INTEL_GVT_OPREGION_PORDER);
+
+        if (!vgpu_opregion(vgpu)->va)
+                return -ENOMEM;
+
+        memcpy_fromio(vgpu_opregion(vgpu)->va, host_va,
+                        INTEL_GVT_OPREGION_SIZE);
+
+        for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
+                vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
+
+        /* for unknown reason, the value in LID field is incorrect
+         * which block the windows guest, so workaround it by force
+         * setting it to "OPEN"
+         */
+        buf = (u8 *)vgpu_opregion(vgpu)->va;
+        buf[INTEL_GVT_OPREGION_CLID] = 0x3;
+
+        return 0;
+}
+
+static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map)
+{
+        u64 mfn;
+        int i, ret;
+
+        for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) {
+                mfn = intel_gvt_hypervisor_virt_to_mfn(vgpu_opregion(vgpu)
+                        + i * PAGE_SIZE);
+                if (mfn == INTEL_GVT_INVALID_ADDR) {
+                        gvt_err("fail to get MFN from VA\n");
+                        return -EINVAL;
+                }
+                ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu,
+                                vgpu_opregion(vgpu)->gfn[i],
+                                mfn, 1, map, GVT_MAP_OPREGION);
+                if (ret) {
+                        gvt_err("fail to map GFN to MFN, errno: %d\n", ret);
+                        return ret;
+                }
+        }
+        return 0;
+}
+
+/**
+ * intel_vgpu_clean_opregion - clean the stuff used to emulate opregion
+ * @vgpu: a vGPU
+ *
+ */
+void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu)
+{
+        int i;
+
+        gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id);
+
+        if (!vgpu_opregion(vgpu)->va)
+                return;
+
+        if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) {
+                vunmap(vgpu_opregion(vgpu)->va);
+                for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) {
+                        if (vgpu_opregion(vgpu)->pages[i]) {
+                                put_page(vgpu_opregion(vgpu)->pages[i]);
+                                vgpu_opregion(vgpu)->pages[i] = NULL;
+                        }
+                }
+        } else {
+                map_vgpu_opregion(vgpu, false);
+                free_pages((unsigned long)vgpu_opregion(vgpu)->va,
+                                INTEL_GVT_OPREGION_PORDER);
+        }
+
+        vgpu_opregion(vgpu)->va = NULL;
+}
+
+/**
+ * intel_vgpu_init_opregion - initialize the stuff used to emulate opregion
+ * @vgpu: a vGPU
+ * @gpa: guest physical address of opregion
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa)
+{
+        int ret;
+
+        gvt_dbg_core("vgpu%d: init vgpu opregion\n", vgpu->id);
+
+        if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) {
+                gvt_dbg_core("emulate opregion from kernel\n");
+
+                ret = init_vgpu_opregion(vgpu, gpa);
+                if (ret)
+                        return ret;
+
+                ret = map_vgpu_opregion(vgpu, true);
+                if (ret)
+                        return ret;
+        } else {
+                gvt_dbg_core("emulate opregion from userspace\n");
+
+                /*
+                 * If opregion pages are not allocated from host kenrel,
+                 * most of the params are meaningless
+                 */
+                ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu,
+                                0, /* not used */
+                                0, /* not used */
+                                2, /* not used */
+                                1,
+                                GVT_MAP_OPREGION);
+                if (ret)
+                        return ret;
+        }
+        return 0;
+}
+
+/**
+ * intel_gvt_clean_opregion - clean host opergion related stuffs
+ * @gvt: a GVT device
+ *
+ */
+void intel_gvt_clean_opregion(struct intel_gvt *gvt)
+{
+        iounmap(gvt->opregion.opregion_va);
+        gvt->opregion.opregion_va = NULL;
+}
+
+/**
+ * intel_gvt_init_opregion - initialize host opergion related stuffs
+ * @gvt: a GVT device
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_init_opregion(struct intel_gvt *gvt)
+{
+        gvt_dbg_core("init host opregion\n");
+
+        pci_read_config_dword(gvt->dev_priv->drm.pdev, INTEL_GVT_PCI_OPREGION,
+                        &gvt->opregion.opregion_pa);
+
+        gvt->opregion.opregion_va = acpi_os_ioremap(gvt->opregion.opregion_pa,
+                        INTEL_GVT_OPREGION_SIZE);
+        if (!gvt->opregion.opregion_va) {
+                gvt_err("fail to map host opregion\n");
+                return -EFAULT;
+        }
+        return 0;
+}
+
+#define GVT_OPREGION_FUNC(scic)                                 \
+        ({                                                      \
+         u32 __ret;                                             \
+         __ret = (scic & OPREGION_SCIC_FUNC_MASK) >>            \
+         OPREGION_SCIC_FUNC_SHIFT;                              \
+         __ret;                                                 \
+         })
+
+#define GVT_OPREGION_SUBFUNC(scic)                              \
+        ({                                                      \
+         u32 __ret;                                             \
+         __ret = (scic & OPREGION_SCIC_SUBFUNC_MASK) >>         \
+         OPREGION_SCIC_SUBFUNC_SHIFT;                           \
+         __ret;                                                 \
+         })
+
+static const char *opregion_func_name(u32 func)
+{
+        const char *name = NULL;
+
+        switch (func) {
+        case 0 ... 3:
+        case 5:
+        case 7 ... 15:
+                name = "Reserved";
+                break;
+
+        case 4:
+                name = "Get BIOS Data";
+                break;
+
+        case 6:
+                name = "System BIOS Callbacks";
+                break;
+
+        default:
+                name = "Unknown";
+                break;
+        }
+        return name;
+}
+
+static const char *opregion_subfunc_name(u32 subfunc)
+{
+        const char *name = NULL;
+
+        switch (subfunc) {
+        case 0:
+                name = "Supported Calls";
+                break;
+
+        case 1:
+                name = "Requested Callbacks";
+                break;
+
+        case 2 ... 3:
+        case 8 ... 9:
+                name = "Reserved";
+                break;
+
+        case 5:
+                name = "Boot Display";
+                break;
+
+        case 6:
+                name = "TV-Standard/Video-Connector";
+                break;
+
+        case 7:
+                name = "Internal Graphics";
+                break;
+
+        case 10:
+                name = "Spread Spectrum Clocks";
+                break;
+
+        case 11:
+                name = "Get AKSV";
+                break;
+
+        default:
+                name = "Unknown";
+                break;
+        }
+        return name;
+};
+
+static bool querying_capabilities(u32 scic)
+{
+        u32 func, subfunc;
+
+        func = GVT_OPREGION_FUNC(scic);
+        subfunc = GVT_OPREGION_SUBFUNC(scic);
+
+        if ((func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA &&
+                subfunc == INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS)
+                || (func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA &&
+                 subfunc == INTEL_GVT_OPREGION_SCIC_SF_REQEUSTEDCALLBACKS)
+                || (func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSCALLBACKS &&
+                 subfunc == INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS)) {
+                return true;
+        }
+        return false;
+}
+
+/**
+ * intel_vgpu_emulate_opregion_request - emulating OpRegion request
+ * @vgpu: a vGPU
+ * @swsci: SWSCI request
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci)
+{
+        u32 *scic, *parm;
+        u32 func, subfunc;
+
+        scic = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_SCIC;
+        parm = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_PARM;
+
+        if (!(swsci & SWSCI_SCI_SELECT)) {
+                gvt_err("vgpu%d: requesting SMI service\n", vgpu->id);
+                return 0;
+        }
+        /* ignore non 0->1 trasitions */
+        if ((vgpu_cfg_space(vgpu)[INTEL_GVT_PCI_SWSCI]
+                                & SWSCI_SCI_TRIGGER) ||
+                        !(swsci & SWSCI_SCI_TRIGGER)) {
+                return 0;
+        }
+
+        func = GVT_OPREGION_FUNC(*scic);
+        subfunc = GVT_OPREGION_SUBFUNC(*scic);
+        if (!querying_capabilities(*scic)) {
+                gvt_err("vgpu%d: requesting runtime service: func \"%s\","
+                                " subfunc \"%s\"\n",
+                                vgpu->id,
+                                opregion_func_name(func),
+                                opregion_subfunc_name(subfunc));
+                /*
+                 * emulate exit status of function call, '0' means
+                 * "failure, generic, unsupported or unknown cause"
+                 */
+                *scic &= ~OPREGION_SCIC_EXIT_MASK;
+                return 0;
+        }
+
+        *scic = 0;
+        *parm = 0;
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
new file mode 100644
index 000000000000..0dfe789d8f02
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GVT_REG_H
+#define _GVT_REG_H
+
+#define INTEL_GVT_PCI_CLASS_VGA_OTHER   0x80
+
+#define INTEL_GVT_PCI_GMCH_CONTROL      0x50
+#define   BDW_GMCH_GMS_SHIFT            8
+#define   BDW_GMCH_GMS_MASK             0xff
+
+#define INTEL_GVT_PCI_SWSCI             0xe8
+#define   SWSCI_SCI_SELECT              (1 << 15)
+#define   SWSCI_SCI_TRIGGER             1
+
+#define INTEL_GVT_PCI_OPREGION          0xfc
+
+#define INTEL_GVT_OPREGION_CLID         0x1AC
+#define INTEL_GVT_OPREGION_SCIC         0x200
+#define   OPREGION_SCIC_FUNC_MASK       0x1E
+#define   OPREGION_SCIC_FUNC_SHIFT      1
+#define   OPREGION_SCIC_SUBFUNC_MASK    0xFF00
+#define   OPREGION_SCIC_SUBFUNC_SHIFT   8
+#define   OPREGION_SCIC_EXIT_MASK       0xE0
+#define INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA         4
+#define INTEL_GVT_OPREGION_SCIC_F_GETBIOSCALLBACKS    6
+#define INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS      0
+#define INTEL_GVT_OPREGION_SCIC_SF_REQEUSTEDCALLBACKS 1
+#define INTEL_GVT_OPREGION_PARM                   0x204
+
+#define INTEL_GVT_OPREGION_PAGES        2
+#define INTEL_GVT_OPREGION_PORDER       1
+#define INTEL_GVT_OPREGION_SIZE         (2 * 4096)
+
+#define VGT_SPRSTRIDE(pipe)     _PIPE(pipe, _SPRA_STRIDE, _PLANE_STRIDE_2_B)
+
+#define _REG_VECS_EXCC          0x1A028
+#define _REG_VCS2_EXCC          0x1c028
+
+#define _REG_701C0(pipe, plane) (0x701c0 + pipe * 0x1000 + (plane - 1) * 0x100)
+#define _REG_701C4(pipe, plane) (0x701c4 + pipe * 0x1000 + (plane - 1) * 0x100)
+
+#define GFX_MODE_BIT_SET_IN_MASK(val, bit) \
+                ((((bit) & 0xffff0000) == 0) && !!((val) & (((bit) << 16))))
+
+#define FORCEWAKE_RENDER_GEN9_REG 0xa278
+#define FORCEWAKE_ACK_RENDER_GEN9_REG 0x0D84
+#define FORCEWAKE_BLITTER_GEN9_REG 0xa188
+#define FORCEWAKE_ACK_BLITTER_GEN9_REG 0x130044
+#define FORCEWAKE_MEDIA_GEN9_REG 0xa270
+#define FORCEWAKE_ACK_MEDIA_GEN9_REG 0x0D88
+#define FORCEWAKE_ACK_HSW_REG 0x130044
+
+#define RB_HEAD_OFF_MASK        ((1U << 21) - (1U << 2))
+#define RB_TAIL_OFF_MASK        ((1U << 21) - (1U << 3))
+#define RB_TAIL_SIZE_MASK       ((1U << 21) - (1U << 12))
+#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + GTT_PAGE_SIZE)
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c
new file mode 100644
index 000000000000..feebb65ba641
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/render.c
@@ -0,0 +1,291 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+struct render_mmio {
+        int ring_id;
+        i915_reg_t reg;
+        u32 mask;
+        bool in_context;
+        u32 value;
+};
+
+static struct render_mmio gen8_render_mmio_list[] = {
+        {RCS, _MMIO(0x229c), 0xffff, false},
+        {RCS, _MMIO(0x2248), 0x0, false},
+        {RCS, _MMIO(0x2098), 0x0, false},
+        {RCS, _MMIO(0x20c0), 0xffff, true},
+        {RCS, _MMIO(0x24d0), 0, false},
+        {RCS, _MMIO(0x24d4), 0, false},
+        {RCS, _MMIO(0x24d8), 0, false},
+        {RCS, _MMIO(0x24dc), 0, false},
+        {RCS, _MMIO(0x7004), 0xffff, true},
+        {RCS, _MMIO(0x7008), 0xffff, true},
+        {RCS, _MMIO(0x7000), 0xffff, true},
+        {RCS, _MMIO(0x7010), 0xffff, true},
+        {RCS, _MMIO(0x7300), 0xffff, true},
+        {RCS, _MMIO(0x83a4), 0xffff, true},
+
+        {BCS, _MMIO(0x2229c), 0xffff, false},
+        {BCS, _MMIO(0x2209c), 0xffff, false},
+        {BCS, _MMIO(0x220c0), 0xffff, false},
+        {BCS, _MMIO(0x22098), 0x0, false},
+        {BCS, _MMIO(0x22028), 0x0, false},
+};
+
+static struct render_mmio gen9_render_mmio_list[] = {
+        {RCS, _MMIO(0x229c), 0xffff, false},
+        {RCS, _MMIO(0x2248), 0x0, false},
+        {RCS, _MMIO(0x2098), 0x0, false},
+        {RCS, _MMIO(0x20c0), 0xffff, true},
+        {RCS, _MMIO(0x24d0), 0, false},
+        {RCS, _MMIO(0x24d4), 0, false},
+        {RCS, _MMIO(0x24d8), 0, false},
+        {RCS, _MMIO(0x24dc), 0, false},
+        {RCS, _MMIO(0x7004), 0xffff, true},
+        {RCS, _MMIO(0x7008), 0xffff, true},
+        {RCS, _MMIO(0x7000), 0xffff, true},
+        {RCS, _MMIO(0x7010), 0xffff, true},
+        {RCS, _MMIO(0x7300), 0xffff, true},
+        {RCS, _MMIO(0x83a4), 0xffff, true},
+
+        {RCS, _MMIO(0x40e0), 0, false},
+        {RCS, _MMIO(0x40e4), 0, false},
+        {RCS, _MMIO(0x2580), 0xffff, true},
+        {RCS, _MMIO(0x7014), 0xffff, true},
+        {RCS, _MMIO(0x20ec), 0xffff, false},
+        {RCS, _MMIO(0xb118), 0, false},
+        {RCS, _MMIO(0xe100), 0xffff, true},
+        {RCS, _MMIO(0xe180), 0xffff, true},
+        {RCS, _MMIO(0xe184), 0xffff, true},
+        {RCS, _MMIO(0xe188), 0xffff, true},
+        {RCS, _MMIO(0xe194), 0xffff, true},
+        {RCS, _MMIO(0x4de0), 0, false},
+        {RCS, _MMIO(0x4de4), 0, false},
+        {RCS, _MMIO(0x4de8), 0, false},
+        {RCS, _MMIO(0x4dec), 0, false},
+        {RCS, _MMIO(0x4df0), 0, false},
+        {RCS, _MMIO(0x4df4), 0, false},
+
+        {BCS, _MMIO(0x2229c), 0xffff, false},
+        {BCS, _MMIO(0x2209c), 0xffff, false},
+        {BCS, _MMIO(0x220c0), 0xffff, false},
+        {BCS, _MMIO(0x22098), 0x0, false},
+        {BCS, _MMIO(0x22028), 0x0, false},
+
+        {VCS2, _MMIO(0x1c028), 0xffff, false},
+
+        {VECS, _MMIO(0x1a028), 0xffff, false},
+};
+
+static u32 gen9_render_mocs[I915_NUM_ENGINES][64];
+static u32 gen9_render_mocs_L3[32];
+
+static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        i915_reg_t reg;
+        u32 regs[] = {
+                [RCS] = 0x4260,
+                [VCS] = 0x4264,
+                [VCS2] = 0x4268,
+                [BCS] = 0x426c,
+                [VECS] = 0x4270,
+        };
+
+        if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+                return;
+
+        if (!test_and_clear_bit(ring_id, (void *)vgpu->tlb_handle_pending))
+                return;
+
+        reg = _MMIO(regs[ring_id]);
+
+        I915_WRITE(reg, 0x1);
+
+        if (wait_for_atomic((I915_READ(reg) == 0), 50))
+                gvt_err("timeout in invalidate ring (%d) tlb\n", ring_id);
+
+        gvt_dbg_core("invalidate TLB for ring %d\n", ring_id);
+}
+
+static void load_mocs(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        i915_reg_t offset, l3_offset;
+        u32 regs[] = {
+                [RCS] = 0xc800,
+                [VCS] = 0xc900,
+                [VCS2] = 0xca00,
+                [BCS] = 0xcc00,
+                [VECS] = 0xcb00,
+        };
+        int i;
+
+        if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+                return;
+
+        if (!IS_SKYLAKE(dev_priv))
+                return;
+
+        for (i = 0; i < 64; i++) {
+                gen9_render_mocs[ring_id][i] = I915_READ(offset);
+                I915_WRITE(offset, vgpu_vreg(vgpu, offset));
+                POSTING_READ(offset);
+                offset.reg += 4;
+        }
+
+        if (ring_id == RCS) {
+                l3_offset.reg = 0xb020;
+                for (i = 0; i < 32; i++) {
+                        gen9_render_mocs_L3[i] = I915_READ(l3_offset);
+                        I915_WRITE(l3_offset, vgpu_vreg(vgpu, offset));
+                        POSTING_READ(l3_offset);
+                        l3_offset.reg += 4;
+                }
+        }
+}
+
+static void restore_mocs(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        i915_reg_t offset, l3_offset;
+        u32 regs[] = {
+                [RCS] = 0xc800,
+                [VCS] = 0xc900,
+                [VCS2] = 0xca00,
+                [BCS] = 0xcc00,
+                [VECS] = 0xcb00,
+        };
+        int i;
+
+        if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+                return;
+
+        if (!IS_SKYLAKE(dev_priv))
+                return;
+
+        for (i = 0; i < 64; i++) {
+                vgpu_vreg(vgpu, offset) = I915_READ(offset);
+                I915_WRITE(offset, gen9_render_mocs[ring_id][i]);
+                POSTING_READ(offset);
+                offset.reg += 4;
+        }
+
+        if (ring_id == RCS) {
+                l3_offset.reg = 0xb020;
+                for (i = 0; i < 32; i++) {
+                        vgpu_vreg(vgpu, l3_offset) = I915_READ(l3_offset);
+                        I915_WRITE(l3_offset, gen9_render_mocs_L3[i]);
+                        POSTING_READ(l3_offset);
+                        l3_offset.reg += 4;
+                }
+        }
+}
+
+void intel_gvt_load_render_mmio(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct render_mmio *mmio;
+        u32 v;
+        int i, array_size;
+
+        if (IS_SKYLAKE(vgpu->gvt->dev_priv)) {
+                mmio = gen9_render_mmio_list;
+                array_size = ARRAY_SIZE(gen9_render_mmio_list);
+                load_mocs(vgpu, ring_id);
+        } else {
+                mmio = gen8_render_mmio_list;
+                array_size = ARRAY_SIZE(gen8_render_mmio_list);
+        }
+
+        for (i = 0; i < array_size; i++, mmio++) {
+                if (mmio->ring_id != ring_id)
+                        continue;
+
+                mmio->value = I915_READ(mmio->reg);
+                if (mmio->mask)
+                        v = vgpu_vreg(vgpu, mmio->reg) | (mmio->mask << 16);
+                else
+                        v = vgpu_vreg(vgpu, mmio->reg);
+
+                I915_WRITE(mmio->reg, v);
+                POSTING_READ(mmio->reg);
+
+                gvt_dbg_render("load reg %x old %x new %x\n",
+                                i915_mmio_reg_offset(mmio->reg),
+                                mmio->value, v);
+        }
+        handle_tlb_pending_event(vgpu, ring_id);
+}
+
+void intel_gvt_restore_render_mmio(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct render_mmio *mmio;
+        u32 v;
+        int i, array_size;
+
+        if (IS_SKYLAKE(dev_priv)) {
+                mmio = gen9_render_mmio_list;
+                array_size = ARRAY_SIZE(gen9_render_mmio_list);
+                restore_mocs(vgpu, ring_id);
+        } else {
+                mmio = gen8_render_mmio_list;
+                array_size = ARRAY_SIZE(gen8_render_mmio_list);
+        }
+
+        for (i = 0; i < array_size; i++, mmio++) {
+                if (mmio->ring_id != ring_id)
+                        continue;
+
+                vgpu_vreg(vgpu, mmio->reg) = I915_READ(mmio->reg);
+
+                if (mmio->mask) {
+                        vgpu_vreg(vgpu, mmio->reg) &= ~(mmio->mask << 16);
+                        v = mmio->value | (mmio->mask << 16);
+                } else
+                        v = mmio->value;
+
+                I915_WRITE(mmio->reg, v);
+                POSTING_READ(mmio->reg);
+
+                gvt_dbg_render("restore reg %x old %x new %x\n",
+                                i915_mmio_reg_offset(mmio->reg),
+                                mmio->value, v);
+        }
+}
diff --git a/drivers/gpu/drm/i915/gvt/render.h b/drivers/gpu/drm/i915/gvt/render.h
new file mode 100644
index 000000000000..dac1a3cc458b
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/render.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#ifndef __GVT_RENDER_H__
+#define __GVT_RENDER_H__
+
+void intel_gvt_load_render_mmio(struct intel_vgpu *vgpu, int ring_id);
+
+void intel_gvt_restore_render_mmio(struct intel_vgpu *vgpu, int ring_id);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c
new file mode 100644
index 000000000000..1df6a5460f3e
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.c
@@ -0,0 +1,294 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Anhua Xu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu)
+{
+        struct intel_vgpu_execlist *execlist;
+        enum intel_engine_id i;
+        struct intel_engine_cs *engine;
+
+        for_each_engine(engine, vgpu->gvt->dev_priv, i) {
+                execlist = &vgpu->execlist[i];
+                if (!list_empty(workload_q_head(vgpu, i)))
+                        return true;
+        }
+
+        return false;
+}
+
+static void try_to_schedule_next_vgpu(struct intel_gvt *gvt)
+{
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+        enum intel_engine_id i;
+        struct intel_engine_cs *engine;
+
+        /* no target to schedule */
+        if (!scheduler->next_vgpu)
+                return;
+
+        gvt_dbg_sched("try to schedule next vgpu %d\n",
+                        scheduler->next_vgpu->id);
+
+        /*
+         * after the flag is set, workload dispatch thread will
+         * stop dispatching workload for current vgpu
+         */
+        scheduler->need_reschedule = true;
+
+        /* still have uncompleted workload? */
+        for_each_engine(engine, gvt->dev_priv, i) {
+                if (scheduler->current_workload[i]) {
+                        gvt_dbg_sched("still have running workload\n");
+                        return;
+                }
+        }
+
+        gvt_dbg_sched("switch to next vgpu %d\n",
+                        scheduler->next_vgpu->id);
+
+        /* switch current vgpu */
+        scheduler->current_vgpu = scheduler->next_vgpu;
+        scheduler->next_vgpu = NULL;
+
+        scheduler->need_reschedule = false;
+
+        /* wake up workload dispatch thread */
+        for_each_engine(engine, gvt->dev_priv, i)
+                wake_up(&scheduler->waitq[i]);
+}
+
+struct tbs_vgpu_data {
+        struct list_head list;
+        struct intel_vgpu *vgpu;
+        /* put some per-vgpu sched stats here */
+};
+
+struct tbs_sched_data {
+        struct intel_gvt *gvt;
+        struct delayed_work work;
+        unsigned long period;
+        struct list_head runq_head;
+};
+
+#define GVT_DEFAULT_TIME_SLICE (1 * HZ / 1000)
+
+static void tbs_sched_func(struct work_struct *work)
+{
+        struct tbs_sched_data *sched_data = container_of(work,
+                        struct tbs_sched_data, work.work);
+        struct tbs_vgpu_data *vgpu_data;
+
+        struct intel_gvt *gvt = sched_data->gvt;
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+
+        struct intel_vgpu *vgpu = NULL;
+        struct list_head *pos, *head;
+
+        mutex_lock(&gvt->lock);
+
+        /* no vgpu or has already had a target */
+        if (list_empty(&sched_data->runq_head) || scheduler->next_vgpu)
+                goto out;
+
+        if (scheduler->current_vgpu) {
+                vgpu_data = scheduler->current_vgpu->sched_data;
+                head = &vgpu_data->list;
+        } else {
+                gvt_dbg_sched("no current vgpu search from q head\n");
+                head = &sched_data->runq_head;
+        }
+
+        /* search a vgpu with pending workload */
+        list_for_each(pos, head) {
+                if (pos == &sched_data->runq_head)
+                        continue;
+
+                vgpu_data = container_of(pos, struct tbs_vgpu_data, list);
+                if (!vgpu_has_pending_workload(vgpu_data->vgpu))
+                        continue;
+
+                vgpu = vgpu_data->vgpu;
+                break;
+        }
+
+        if (vgpu) {
+                scheduler->next_vgpu = vgpu;
+                gvt_dbg_sched("pick next vgpu %d\n", vgpu->id);
+        }
+out:
+        if (scheduler->next_vgpu) {
+                gvt_dbg_sched("try to schedule next vgpu %d\n",
+                                scheduler->next_vgpu->id);
+                try_to_schedule_next_vgpu(gvt);
+        }
+
+        /*
+         * still have vgpu on runq
+         * or last schedule haven't finished due to running workload
+         */
+        if (!list_empty(&sched_data->runq_head) || scheduler->next_vgpu)
+                schedule_delayed_work(&sched_data->work, sched_data->period);
+
+        mutex_unlock(&gvt->lock);
+}
+
+static int tbs_sched_init(struct intel_gvt *gvt)
+{
+        struct intel_gvt_workload_scheduler *scheduler =
+                &gvt->scheduler;
+
+        struct tbs_sched_data *data;
+
+        data = kzalloc(sizeof(*data), GFP_KERNEL);
+        if (!data)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&data->runq_head);
+        INIT_DELAYED_WORK(&data->work, tbs_sched_func);
+        data->period = GVT_DEFAULT_TIME_SLICE;
+        data->gvt = gvt;
+
+        scheduler->sched_data = data;
+        return 0;
+}
+
+static void tbs_sched_clean(struct intel_gvt *gvt)
+{
+        struct intel_gvt_workload_scheduler *scheduler =
+                &gvt->scheduler;
+        struct tbs_sched_data *data = scheduler->sched_data;
+
+        cancel_delayed_work(&data->work);
+        kfree(data);
+        scheduler->sched_data = NULL;
+}
+
+static int tbs_sched_init_vgpu(struct intel_vgpu *vgpu)
+{
+        struct tbs_vgpu_data *data;
+
+        data = kzalloc(sizeof(*data), GFP_KERNEL);
+        if (!data)
+                return -ENOMEM;
+
+        data->vgpu = vgpu;
+        INIT_LIST_HEAD(&data->list);
+
+        vgpu->sched_data = data;
+        return 0;
+}
+
+static void tbs_sched_clean_vgpu(struct intel_vgpu *vgpu)
+{
+        kfree(vgpu->sched_data);
+        vgpu->sched_data = NULL;
+}
+
+static void tbs_sched_start_schedule(struct intel_vgpu *vgpu)
+{
+        struct tbs_sched_data *sched_data = vgpu->gvt->scheduler.sched_data;
+        struct tbs_vgpu_data *vgpu_data = vgpu->sched_data;
+
+        if (!list_empty(&vgpu_data->list))
+                return;
+
+        list_add_tail(&vgpu_data->list, &sched_data->runq_head);
+        schedule_delayed_work(&sched_data->work, sched_data->period);
+}
+
+static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu)
+{
+        struct tbs_vgpu_data *vgpu_data = vgpu->sched_data;
+
+        list_del_init(&vgpu_data->list);
+}
+
+static struct intel_gvt_sched_policy_ops tbs_schedule_ops = {
+        .init = tbs_sched_init,
+        .clean = tbs_sched_clean,
+        .init_vgpu = tbs_sched_init_vgpu,
+        .clean_vgpu = tbs_sched_clean_vgpu,
+        .start_schedule = tbs_sched_start_schedule,
+        .stop_schedule = tbs_sched_stop_schedule,
+};
+
+int intel_gvt_init_sched_policy(struct intel_gvt *gvt)
+{
+        gvt->scheduler.sched_ops = &tbs_schedule_ops;
+
+        return gvt->scheduler.sched_ops->init(gvt);
+}
+
+void intel_gvt_clean_sched_policy(struct intel_gvt *gvt)
+{
+        gvt->scheduler.sched_ops->clean(gvt);
+}
+
+int intel_vgpu_init_sched_policy(struct intel_vgpu *vgpu)
+{
+        return vgpu->gvt->scheduler.sched_ops->init_vgpu(vgpu);
+}
+
+void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu)
+{
+        vgpu->gvt->scheduler.sched_ops->clean_vgpu(vgpu);
+}
+
+void intel_vgpu_start_schedule(struct intel_vgpu *vgpu)
+{
+        gvt_dbg_core("vgpu%d: start schedule\n", vgpu->id);
+
+        vgpu->gvt->scheduler.sched_ops->start_schedule(vgpu);
+}
+
+void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt_workload_scheduler *scheduler =
+                &vgpu->gvt->scheduler;
+
+        gvt_dbg_core("vgpu%d: stop schedule\n", vgpu->id);
+
+        scheduler->sched_ops->stop_schedule(vgpu);
+
+        if (scheduler->next_vgpu == vgpu)
+                scheduler->next_vgpu = NULL;
+
+        if (scheduler->current_vgpu == vgpu) {
+                /* stop workload dispatching */
+                scheduler->need_reschedule = true;
+                scheduler->current_vgpu = NULL;
+        }
+}
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.h b/drivers/gpu/drm/i915/gvt/sched_policy.h
new file mode 100644
index 000000000000..bb8b9097e41a
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Anhua Xu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef __GVT_SCHED_POLICY__
+#define __GVT_SCHED_POLICY__
+
+struct intel_gvt_sched_policy_ops {
+        int (*init)(struct intel_gvt *gvt);
+        void (*clean)(struct intel_gvt *gvt);
+        int (*init_vgpu)(struct intel_vgpu *vgpu);
+        void (*clean_vgpu)(struct intel_vgpu *vgpu);
+        void (*start_schedule)(struct intel_vgpu *vgpu);
+        void (*stop_schedule)(struct intel_vgpu *vgpu);
+};
+
+int intel_gvt_init_sched_policy(struct intel_gvt *gvt);
+
+void intel_gvt_clean_sched_policy(struct intel_gvt *gvt);
+
+int intel_vgpu_init_sched_policy(struct intel_vgpu *vgpu);
+
+void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu);
+
+void intel_vgpu_start_schedule(struct intel_vgpu *vgpu);
+
+void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
new file mode 100644
index 000000000000..e96eaeebeb0a
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -0,0 +1,578 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Chanbin Du <changbin.du@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *
+ */
+
+#include <linux/kthread.h>
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define RING_CTX_OFF(x) \
+        offsetof(struct execlist_ring_context, x)
+
+static void set_context_pdp_root_pointer(
+                struct execlist_ring_context *ring_context,
+                u32 pdp[8])
+{
+        struct execlist_mmio_pair *pdp_pair = &ring_context->pdp3_UDW;
+        int i;
+
+        for (i = 0; i < 8; i++)
+                pdp_pair[i].val = pdp[7 - i];
+}
+
+static int populate_shadow_context(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        int ring_id = workload->ring_id;
+        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+        struct drm_i915_gem_object *ctx_obj =
+                shadow_ctx->engine[ring_id].state->obj;
+        struct execlist_ring_context *shadow_ring_context;
+        struct page *page;
+        void *dst;
+        unsigned long context_gpa, context_page_num;
+        int i;
+
+        gvt_dbg_sched("ring id %d workload lrca %x", ring_id,
+                        workload->ctx_desc.lrca);
+
+        context_page_num = intel_lr_context_size(
+                        gvt->dev_priv->engine[ring_id]);
+
+        context_page_num = context_page_num >> PAGE_SHIFT;
+
+        if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
+                context_page_num = 19;
+
+        i = 2;
+
+        while (i < context_page_num) {
+                context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+                                (u32)((workload->ctx_desc.lrca + i) <<
+                                GTT_PAGE_SHIFT));
+                if (context_gpa == INTEL_GVT_INVALID_ADDR) {
+                        gvt_err("Invalid guest context descriptor\n");
+                        return -EINVAL;
+                }
+
+                page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
+                dst = kmap_atomic(page);
+                intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
+                                GTT_PAGE_SIZE);
+                kunmap_atomic(dst);
+                i++;
+        }
+
+        page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+        shadow_ring_context = kmap_atomic(page);
+
+#define COPY_REG(name) \
+        intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
+                + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
+
+        COPY_REG(ctx_ctrl);
+        COPY_REG(ctx_timestamp);
+
+        if (ring_id == RCS) {
+                COPY_REG(bb_per_ctx_ptr);
+                COPY_REG(rcs_indirect_ctx);
+                COPY_REG(rcs_indirect_ctx_offset);
+        }
+#undef COPY_REG
+
+        set_context_pdp_root_pointer(shadow_ring_context,
+                                     workload->shadow_mm->shadow_page_table);
+
+        intel_gvt_hypervisor_read_gpa(vgpu,
+                        workload->ring_context_gpa +
+                        sizeof(*shadow_ring_context),
+                        (void *)shadow_ring_context +
+                        sizeof(*shadow_ring_context),
+                        GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+
+        kunmap_atomic(shadow_ring_context);
+        return 0;
+}
+
+static int shadow_context_status_change(struct notifier_block *nb,
+                unsigned long action, void *data)
+{
+        struct intel_vgpu *vgpu = container_of(nb,
+                        struct intel_vgpu, shadow_ctx_notifier_block);
+        struct drm_i915_gem_request *req =
+                (struct drm_i915_gem_request *)data;
+        struct intel_gvt_workload_scheduler *scheduler =
+                &vgpu->gvt->scheduler;
+        struct intel_vgpu_workload *workload =
+                scheduler->current_workload[req->engine->id];
+
+        switch (action) {
+        case INTEL_CONTEXT_SCHEDULE_IN:
+                intel_gvt_load_render_mmio(workload->vgpu,
+                                           workload->ring_id);
+                atomic_set(&workload->shadow_ctx_active, 1);
+                break;
+        case INTEL_CONTEXT_SCHEDULE_OUT:
+                intel_gvt_restore_render_mmio(workload->vgpu,
+                                              workload->ring_id);
+                atomic_set(&workload->shadow_ctx_active, 0);
+                break;
+        default:
+                WARN_ON(1);
+                return NOTIFY_OK;
+        }
+        wake_up(&workload->shadow_ctx_status_wq);
+        return NOTIFY_OK;
+}
+
+static int dispatch_workload(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        int ring_id = workload->ring_id;
+        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+        struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
+        struct drm_i915_gem_request *rq;
+        int ret;
+
+        gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
+                ring_id, workload);
+
+        shadow_ctx->desc_template = workload->ctx_desc.addressing_mode <<
+                                    GEN8_CTX_ADDRESSING_MODE_SHIFT;
+
+        rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
+        if (IS_ERR(rq)) {
+                gvt_err("fail to allocate gem request\n");
+                workload->status = PTR_ERR(rq);
+                return workload->status;
+        }
+
+        gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq);
+
+        workload->req = i915_gem_request_get(rq);
+
+        mutex_lock(&gvt->lock);
+
+        ret = intel_gvt_scan_and_shadow_workload(workload);
+        if (ret)
+                goto err;
+
+        ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
+        if (ret)
+                goto err;
+
+        ret = populate_shadow_context(workload);
+        if (ret)
+                goto err;
+
+        if (workload->prepare) {
+                ret = workload->prepare(workload);
+                if (ret)
+                        goto err;
+        }
+
+        mutex_unlock(&gvt->lock);
+
+        gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
+                        ring_id, workload->req);
+
+        i915_add_request_no_flush(rq);
+        workload->dispatched = true;
+        return 0;
+err:
+        workload->status = ret;
+
+        mutex_unlock(&gvt->lock);
+
+        i915_add_request_no_flush(rq);
+        return ret;
+}
+
+static struct intel_vgpu_workload *pick_next_workload(
+                struct intel_gvt *gvt, int ring_id)
+{
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+        struct intel_vgpu_workload *workload = NULL;
+
+        mutex_lock(&gvt->lock);
+
+        /*
+         * no current vgpu / will be scheduled out / no workload
+         * bail out
+         */
+        if (!scheduler->current_vgpu) {
+                gvt_dbg_sched("ring id %d stop - no current vgpu\n", ring_id);
+                goto out;
+        }
+
+        if (scheduler->need_reschedule) {
+                gvt_dbg_sched("ring id %d stop - will reschedule\n", ring_id);
+                goto out;
+        }
+
+        if (list_empty(workload_q_head(scheduler->current_vgpu, ring_id))) {
+                gvt_dbg_sched("ring id %d stop - no available workload\n",
+                                ring_id);
+                goto out;
+        }
+
+        /*
+         * still have current workload, maybe the workload disptacher
+         * fail to submit it for some reason, resubmit it.
+         */
+        if (scheduler->current_workload[ring_id]) {
+                workload = scheduler->current_workload[ring_id];
+                gvt_dbg_sched("ring id %d still have current workload %p\n",
+                                ring_id, workload);
+                goto out;
+        }
+
+        /*
+         * pick a workload as current workload
+         * once current workload is set, schedule policy routines
+         * will wait the current workload is finished when trying to
+         * schedule out a vgpu.
+         */
+        scheduler->current_workload[ring_id] = container_of(
+                        workload_q_head(scheduler->current_vgpu, ring_id)->next,
+                        struct intel_vgpu_workload, list);
+
+        workload = scheduler->current_workload[ring_id];
+
+        gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload);
+
+        atomic_inc(&workload->vgpu->running_workload_num);
+out:
+        mutex_unlock(&gvt->lock);
+        return workload;
+}
+
+static void update_guest_context(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_gvt *gvt = vgpu->gvt;
+        int ring_id = workload->ring_id;
+        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+        struct drm_i915_gem_object *ctx_obj =
+                shadow_ctx->engine[ring_id].state->obj;
+        struct execlist_ring_context *shadow_ring_context;
+        struct page *page;
+        void *src;
+        unsigned long context_gpa, context_page_num;
+        int i;
+
+        gvt_dbg_sched("ring id %d workload lrca %x\n", ring_id,
+                        workload->ctx_desc.lrca);
+
+        context_page_num = intel_lr_context_size(
+                        gvt->dev_priv->engine[ring_id]);
+
+        context_page_num = context_page_num >> PAGE_SHIFT;
+
+        if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
+                context_page_num = 19;
+
+        i = 2;
+
+        while (i < context_page_num) {
+                context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+                                (u32)((workload->ctx_desc.lrca + i) <<
+                                        GTT_PAGE_SHIFT));
+                if (context_gpa == INTEL_GVT_INVALID_ADDR) {
+                        gvt_err("invalid guest context descriptor\n");
+                        return;
+                }
+
+                page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
+                src = kmap_atomic(page);
+                intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
+                                GTT_PAGE_SIZE);
+                kunmap_atomic(src);
+                i++;
+        }
+
+        intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +
+                RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);
+
+        page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+        shadow_ring_context = kmap_atomic(page);
+
+#define COPY_REG(name) \
+        intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
+                RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
+
+        COPY_REG(ctx_ctrl);
+        COPY_REG(ctx_timestamp);
+
+#undef COPY_REG
+
+        intel_gvt_hypervisor_write_gpa(vgpu,
+                        workload->ring_context_gpa +
+                        sizeof(*shadow_ring_context),
+                        (void *)shadow_ring_context +
+                        sizeof(*shadow_ring_context),
+                        GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+
+        kunmap_atomic(shadow_ring_context);
+}
+
+static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
+{
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+        struct intel_vgpu_workload *workload;
+        int event;
+
+        mutex_lock(&gvt->lock);
+
+        workload = scheduler->current_workload[ring_id];
+
+        if (!workload->status && !workload->vgpu->resetting) {
+                wait_event(workload->shadow_ctx_status_wq,
+                           !atomic_read(&workload->shadow_ctx_active));
+
+                update_guest_context(workload);
+
+                for_each_set_bit(event, workload->pending_events,
+                                 INTEL_GVT_EVENT_MAX)
+                        intel_vgpu_trigger_virtual_event(workload->vgpu,
+                                        event);
+        }
+
+        gvt_dbg_sched("ring id %d complete workload %p status %d\n",
+                        ring_id, workload, workload->status);
+
+        scheduler->current_workload[ring_id] = NULL;
+
+        atomic_dec(&workload->vgpu->running_workload_num);
+
+        list_del_init(&workload->list);
+        workload->complete(workload);
+
+        wake_up(&scheduler->workload_complete_wq);
+        mutex_unlock(&gvt->lock);
+}
+
+struct workload_thread_param {
+        struct intel_gvt *gvt;
+        int ring_id;
+};
+
+static DEFINE_MUTEX(scheduler_mutex);
+
+static int workload_thread(void *priv)
+{
+        struct workload_thread_param *p = (struct workload_thread_param *)priv;
+        struct intel_gvt *gvt = p->gvt;
+        int ring_id = p->ring_id;
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+        struct intel_vgpu_workload *workload = NULL;
+        int ret;
+        bool need_force_wake = IS_SKYLAKE(gvt->dev_priv);
+
+        kfree(p);
+
+        gvt_dbg_core("workload thread for ring %d started\n", ring_id);
+
+        while (!kthread_should_stop()) {
+                ret = wait_event_interruptible(scheduler->waitq[ring_id],
+                                kthread_should_stop() ||
+                                (workload = pick_next_workload(gvt, ring_id)));
+
+                WARN_ON_ONCE(ret);
+
+                if (kthread_should_stop())
+                        break;
+
+                mutex_lock(&scheduler_mutex);
+
+                gvt_dbg_sched("ring id %d next workload %p vgpu %d\n",
+                                workload->ring_id, workload,
+                                workload->vgpu->id);
+
+                intel_runtime_pm_get(gvt->dev_priv);
+
+                gvt_dbg_sched("ring id %d will dispatch workload %p\n",
+                                workload->ring_id, workload);
+
+                if (need_force_wake)
+                        intel_uncore_forcewake_get(gvt->dev_priv,
+                                        FORCEWAKE_ALL);
+
+                mutex_lock(&gvt->dev_priv->drm.struct_mutex);
+                ret = dispatch_workload(workload);
+                mutex_unlock(&gvt->dev_priv->drm.struct_mutex);
+
+                if (ret) {
+                        gvt_err("fail to dispatch workload, skip\n");
+                        goto complete;
+                }
+
+                gvt_dbg_sched("ring id %d wait workload %p\n",
+                                workload->ring_id, workload);
+
+                workload->status = i915_wait_request(workload->req,
+                                                     0, NULL, NULL);
+                if (workload->status != 0)
+                        gvt_err("fail to wait workload, skip\n");
+
+complete:
+                gvt_dbg_sched("will complete workload %p\n, status: %d\n",
+                                workload, workload->status);
+
+                mutex_lock(&gvt->dev_priv->drm.struct_mutex);
+                complete_current_workload(gvt, ring_id);
+                mutex_unlock(&gvt->dev_priv->drm.struct_mutex);
+
+                i915_gem_request_put(fetch_and_zero(&workload->req));
+
+                if (need_force_wake)
+                        intel_uncore_forcewake_put(gvt->dev_priv,
+                                        FORCEWAKE_ALL);
+
+                intel_runtime_pm_put(gvt->dev_priv);
+
+                mutex_unlock(&scheduler_mutex);
+
+        }
+        return 0;
+}
+
+void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+
+        if (atomic_read(&vgpu->running_workload_num)) {
+                gvt_dbg_sched("wait vgpu idle\n");
+
+                wait_event(scheduler->workload_complete_wq,
+                                !atomic_read(&vgpu->running_workload_num));
+        }
+}
+
+void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt)
+{
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+        int i;
+
+        gvt_dbg_core("clean workload scheduler\n");
+
+        for (i = 0; i < I915_NUM_ENGINES; i++) {
+                if (scheduler->thread[i]) {
+                        kthread_stop(scheduler->thread[i]);
+                        scheduler->thread[i] = NULL;
+                }
+        }
+}
+
+int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
+{
+        struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+        struct workload_thread_param *param = NULL;
+        int ret;
+        int i;
+
+        gvt_dbg_core("init workload scheduler\n");
+
+        init_waitqueue_head(&scheduler->workload_complete_wq);
+
+        for (i = 0; i < I915_NUM_ENGINES; i++) {
+                /* check ring mask at init time */
+                if (!HAS_ENGINE(gvt->dev_priv, i))
+                        continue;
+
+                init_waitqueue_head(&scheduler->waitq[i]);
+
+                param = kzalloc(sizeof(*param), GFP_KERNEL);
+                if (!param) {
+                        ret = -ENOMEM;
+                        goto err;
+                }
+
+                param->gvt = gvt;
+                param->ring_id = i;
+
+                scheduler->thread[i] = kthread_run(workload_thread, param,
+                        "gvt workload %d", i);
+                if (IS_ERR(scheduler->thread[i])) {
+                        gvt_err("fail to create workload thread\n");
+                        ret = PTR_ERR(scheduler->thread[i]);
+                        goto err;
+                }
+        }
+        return 0;
+err:
+        intel_gvt_clean_workload_scheduler(gvt);
+        kfree(param);
+        param = NULL;
+        return ret;
+}
+
+void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+        atomic_notifier_chain_unregister(&vgpu->shadow_ctx->status_notifier,
+                        &vgpu->shadow_ctx_notifier_block);
+
+        mutex_lock(&dev_priv->drm.struct_mutex);
+
+        /* a little hacky to mark as ctx closed */
+        vgpu->shadow_ctx->closed = true;
+        i915_gem_context_put(vgpu->shadow_ctx);
+
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu)
+{
+        atomic_set(&vgpu->running_workload_num, 0);
+
+        vgpu->shadow_ctx = i915_gem_context_create_gvt(
+                        &vgpu->gvt->dev_priv->drm);
+        if (IS_ERR(vgpu->shadow_ctx))
+                return PTR_ERR(vgpu->shadow_ctx);
+
+        vgpu->shadow_ctx->engine[RCS].initialised = true;
+
+        vgpu->shadow_ctx_notifier_block.notifier_call =
+                shadow_context_status_change;
+
+        atomic_notifier_chain_register(&vgpu->shadow_ctx->status_notifier,
+                                       &vgpu->shadow_ctx_notifier_block);
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
new file mode 100644
index 000000000000..3b30c28bff51
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -0,0 +1,139 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Chanbin Du <changbin.du@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *
+ */
+
+#ifndef _GVT_SCHEDULER_H_
+#define _GVT_SCHEDULER_H_
+
+struct intel_gvt_workload_scheduler {
+        struct intel_vgpu *current_vgpu;
+        struct intel_vgpu *next_vgpu;
+        struct intel_vgpu_workload *current_workload[I915_NUM_ENGINES];
+        bool need_reschedule;
+
+        wait_queue_head_t workload_complete_wq;
+        struct task_struct *thread[I915_NUM_ENGINES];
+        wait_queue_head_t waitq[I915_NUM_ENGINES];
+
+        void *sched_data;
+        struct intel_gvt_sched_policy_ops *sched_ops;
+};
+
+#define INDIRECT_CTX_ADDR_MASK 0xffffffc0
+#define INDIRECT_CTX_SIZE_MASK 0x3f
+struct shadow_indirect_ctx {
+        struct drm_i915_gem_object *obj;
+        unsigned long guest_gma;
+        unsigned long shadow_gma;
+        void *shadow_va;
+        uint32_t size;
+};
+
+#define PER_CTX_ADDR_MASK 0xfffff000
+struct shadow_per_ctx {
+        unsigned long guest_gma;
+        unsigned long shadow_gma;
+};
+
+struct intel_shadow_wa_ctx {
+        struct intel_vgpu_workload *workload;
+        struct shadow_indirect_ctx indirect_ctx;
+        struct shadow_per_ctx per_ctx;
+
+};
+
+struct intel_vgpu_workload {
+        struct intel_vgpu *vgpu;
+        int ring_id;
+        struct drm_i915_gem_request *req;
+        /* if this workload has been dispatched to i915? */
+        bool dispatched;
+        int status;
+
+        struct intel_vgpu_mm *shadow_mm;
+
+        /* different submission model may need different handler */
+        int (*prepare)(struct intel_vgpu_workload *);
+        int (*complete)(struct intel_vgpu_workload *);
+        struct list_head list;
+
+        DECLARE_BITMAP(pending_events, INTEL_GVT_EVENT_MAX);
+        void *shadow_ring_buffer_va;
+
+        /* execlist context information */
+        struct execlist_ctx_descriptor_format ctx_desc;
+        struct execlist_ring_context *ring_context;
+        unsigned long rb_head, rb_tail, rb_ctl, rb_start, rb_len;
+        bool restore_inhibit;
+        struct intel_vgpu_elsp_dwords elsp_dwords;
+        bool emulate_schedule_in;
+        atomic_t shadow_ctx_active;
+        wait_queue_head_t shadow_ctx_status_wq;
+        u64 ring_context_gpa;
+
+        /* shadow batch buffer */
+        struct list_head shadow_bb;
+        struct intel_shadow_wa_ctx wa_ctx;
+};
+
+/* Intel shadow batch buffer is a i915 gem object */
+struct intel_shadow_bb_entry {
+        struct list_head list;
+        struct drm_i915_gem_object *obj;
+        void *va;
+        unsigned long len;
+        void *bb_start_cmd_va;
+};
+
+#define workload_q_head(vgpu, ring_id) \
+        (&(vgpu->workload_q_head[ring_id]))
+
+#define queue_workload(workload) do { \
+        list_add_tail(&workload->list, \
+        workload_q_head(workload->vgpu, workload->ring_id)); \
+        wake_up(&workload->vgpu->gvt-> \
+        scheduler.waitq[workload->ring_id]); \
+} while (0)
+
+int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt);
+
+void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt);
+
+void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu);
+
+int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu);
+
+void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h
new file mode 100644
index 000000000000..53a2d10cf3f1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/trace.h
@@ -0,0 +1,286 @@
+/*
+ * Copyright © 2011-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#if !defined(_GVT_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _GVT_TRACE_H_
+
+#include <linux/types.h>
+#include <linux/stringify.h>
+#include <linux/tracepoint.h>
+#include <asm/tsc.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM gvt
+
+TRACE_EVENT(spt_alloc,
+        TP_PROTO(int id, void *spt, int type, unsigned long mfn,
+                unsigned long gpt_gfn),
+
+        TP_ARGS(id, spt, type, mfn, gpt_gfn),
+
+        TP_STRUCT__entry(
+                __field(int, id)
+                __field(void *, spt)
+                __field(int, type)
+                __field(unsigned long, mfn)
+                __field(unsigned long, gpt_gfn)
+                ),
+
+        TP_fast_assign(
+                __entry->id = id;
+                __entry->spt = spt;
+                __entry->type = type;
+                __entry->mfn = mfn;
+                __entry->gpt_gfn = gpt_gfn;
+        ),
+
+        TP_printk("VM%d [alloc] spt %p type %d mfn 0x%lx gfn 0x%lx\n",
+                __entry->id,
+                __entry->spt,
+                __entry->type,
+                __entry->mfn,
+                __entry->gpt_gfn)
+);
+
+TRACE_EVENT(spt_free,
+        TP_PROTO(int id, void *spt, int type),
+
+        TP_ARGS(id, spt, type),
+
+        TP_STRUCT__entry(
+                __field(int, id)
+                __field(void *, spt)
+                __field(int, type)
+                ),
+
+        TP_fast_assign(
+                __entry->id = id;
+                __entry->spt = spt;
+                __entry->type = type;
+        ),
+
+        TP_printk("VM%u [free] spt %p type %d\n",
+                __entry->id,
+                __entry->spt,
+                __entry->type)
+);
+
+#define MAX_BUF_LEN 256
+
+TRACE_EVENT(gma_index,
+        TP_PROTO(const char *prefix, unsigned long gma,
+                unsigned long index),
+
+        TP_ARGS(prefix, gma, index),
+
+        TP_STRUCT__entry(
+                __array(char, buf, MAX_BUF_LEN)
+        ),
+
+        TP_fast_assign(
+                snprintf(__entry->buf, MAX_BUF_LEN,
+                        "%s gma 0x%lx index 0x%lx\n", prefix, gma, index);
+        ),
+
+        TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(gma_translate,
+        TP_PROTO(int id, char *type, int ring_id, int pt_level,
+                unsigned long gma, unsigned long gpa),
+
+        TP_ARGS(id, type, ring_id, pt_level, gma, gpa),
+
+        TP_STRUCT__entry(
+                __array(char, buf, MAX_BUF_LEN)
+        ),
+
+        TP_fast_assign(
+                snprintf(__entry->buf, MAX_BUF_LEN,
+                        "VM%d %s ring %d pt_level %d gma 0x%lx -> gpa 0x%lx\n",
+                                id, type, ring_id, pt_level, gma, gpa);
+        ),
+
+        TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(spt_refcount,
+        TP_PROTO(int id, char *action, void *spt, int before, int after),
+
+        TP_ARGS(id, action, spt, before, after),
+
+        TP_STRUCT__entry(
+                __array(char, buf, MAX_BUF_LEN)
+        ),
+
+        TP_fast_assign(
+                snprintf(__entry->buf, MAX_BUF_LEN,
+                        "VM%d [%s] spt %p before %d -> after %d\n",
+                                id, action, spt, before, after);
+        ),
+
+        TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(spt_change,
+        TP_PROTO(int id, char *action, void *spt, unsigned long gfn,
+                int type),
+
+        TP_ARGS(id, action, spt, gfn, type),
+
+        TP_STRUCT__entry(
+                __array(char, buf, MAX_BUF_LEN)
+        ),
+
+        TP_fast_assign(
+                snprintf(__entry->buf, MAX_BUF_LEN,
+                        "VM%d [%s] spt %p gfn 0x%lx type %d\n",
+                                id, action, spt, gfn, type);
+        ),
+
+        TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(gpt_change,
+        TP_PROTO(int id, const char *tag, void *spt, int type, u64 v,
+                unsigned long index),
+
+        TP_ARGS(id, tag, spt, type, v, index),
+
+        TP_STRUCT__entry(
+                __array(char, buf, MAX_BUF_LEN)
+        ),
+
+        TP_fast_assign(
+                snprintf(__entry->buf, MAX_BUF_LEN,
+                "VM%d [%s] spt %p type %d entry 0x%llx index 0x%lx\n",
+                        id, tag, spt, type, v, index);
+        ),
+
+        TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(oos_change,
+        TP_PROTO(int id, const char *tag, int page_id, void *gpt, int type),
+
+        TP_ARGS(id, tag, page_id, gpt, type),
+
+        TP_STRUCT__entry(
+                __array(char, buf, MAX_BUF_LEN)
+        ),
+
+        TP_fast_assign(
+                snprintf(__entry->buf, MAX_BUF_LEN,
+                "VM%d [oos %s] page id %d gpt %p type %d\n",
+                        id, tag, page_id, gpt, type);
+        ),
+
+        TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(oos_sync,
+        TP_PROTO(int id, int page_id, void *gpt, int type, u64 v,
+                unsigned long index),
+
+        TP_ARGS(id, page_id, gpt, type, v, index),
+
+        TP_STRUCT__entry(
+                __array(char, buf, MAX_BUF_LEN)
+        ),
+
+        TP_fast_assign(
+        snprintf(__entry->buf, MAX_BUF_LEN,
+        "VM%d [oos sync] page id %d gpt %p type %d entry 0x%llx index 0x%lx\n",
+                                id, page_id, gpt, type, v, index);
+        ),
+
+        TP_printk("%s", __entry->buf)
+);
+
+#define MAX_CMD_STR_LEN 256
+TRACE_EVENT(gvt_command,
+                TP_PROTO(u8 vm_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, u32 cmd_len, bool ring_buffer_cmd, cycles_t cost_pre_cmd_handler, cycles_t cost_cmd_handler),
+
+                TP_ARGS(vm_id, ring_id, ip_gma, cmd_va, cmd_len, ring_buffer_cmd, cost_pre_cmd_handler, cost_cmd_handler),
+
+                TP_STRUCT__entry(
+                        __field(u8, vm_id)
+                        __field(u8, ring_id)
+                        __field(int, i)
+                        __array(char, tmp_buf, MAX_CMD_STR_LEN)
+                        __array(char, cmd_str, MAX_CMD_STR_LEN)
+                        ),
+
+                TP_fast_assign(
+                        __entry->vm_id = vm_id;
+                        __entry->ring_id = ring_id;
+                        __entry->cmd_str[0] = '\0';
+                        snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "VM(%d) Ring(%d): %s ip(%08x) pre handler cost (%llu), handler cost (%llu) ", vm_id, ring_id, ring_buffer_cmd ? "RB":"BB", ip_gma, cost_pre_cmd_handler, cost_cmd_handler);
+                        strcat(__entry->cmd_str, __entry->tmp_buf);
+                        entry->i = 0;
+                        while (cmd_len > 0) {
+                                if (cmd_len >= 8) {
+                                        snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x %08x %08x %08x %08x %08x %08x ",
+                                                cmd_va[__entry->i], cmd_va[__entry->i+1], cmd_va[__entry->i+2], cmd_va[__entry->i+3],
+                                                cmd_va[__entry->i+4], cmd_va[__entry->i+5], cmd_va[__entry->i+6], cmd_va[__entry->i+7]);
+                                        __entry->i += 8;
+                                        cmd_len -= 8;
+                                        strcat(__entry->cmd_str, __entry->tmp_buf);
+                                } else if (cmd_len >= 4) {
+                                        snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x %08x %08x ",
+                                                cmd_va[__entry->i], cmd_va[__entry->i+1], cmd_va[__entry->i+2], cmd_va[__entry->i+3]);
+                                        __entry->i += 4;
+                                        cmd_len -= 4;
+                                        strcat(__entry->cmd_str, __entry->tmp_buf);
+                                } else if (cmd_len >= 2) {
+                                        snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x ", cmd_va[__entry->i], cmd_va[__entry->i+1]);
+                                        __entry->i += 2;
+                                        cmd_len -= 2;
+                                        strcat(__entry->cmd_str, __entry->tmp_buf);
+                                } else if (cmd_len == 1) {
+                                        snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x ", cmd_va[__entry->i]);
+                                        __entry->i += 1;
+                                        cmd_len -= 1;
+                                        strcat(__entry->cmd_str, __entry->tmp_buf);
+                                }
+                        }
+                        strcat(__entry->cmd_str, "\n");
+                ),
+
+                TP_printk("%s", __entry->cmd_str)
+);
+#endif /* _GVT_TRACE_H_ */
+
+/* This part must be out of protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/i915/gvt/trace_points.c b/drivers/gpu/drm/i915/gvt/trace_points.c
new file mode 100644
index 000000000000..a3deed692b9c
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/trace_points.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "trace.h"
+
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
new file mode 100644
index 000000000000..9401436d721f
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -0,0 +1,274 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+
+static void clean_vgpu_mmio(struct intel_vgpu *vgpu)
+{
+        vfree(vgpu->mmio.vreg);
+        vgpu->mmio.vreg = vgpu->mmio.sreg = NULL;
+}
+
+static int setup_vgpu_mmio(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        const struct intel_gvt_device_info *info = &gvt->device_info;
+
+        vgpu->mmio.vreg = vzalloc(info->mmio_size * 2);
+        if (!vgpu->mmio.vreg)
+                return -ENOMEM;
+
+        vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size;
+
+        memcpy(vgpu->mmio.vreg, gvt->firmware.mmio, info->mmio_size);
+        memcpy(vgpu->mmio.sreg, gvt->firmware.mmio, info->mmio_size);
+
+        vgpu_vreg(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0;
+
+        /* set the bit 0:2(Core C-State ) to C0 */
+        vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0;
+        return 0;
+}
+
+static void setup_vgpu_cfg_space(struct intel_vgpu *vgpu,
+        struct intel_vgpu_creation_params *param)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+        const struct intel_gvt_device_info *info = &gvt->device_info;
+        u16 *gmch_ctl;
+        int i;
+
+        memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space,
+               info->cfg_space_size);
+
+        if (!param->primary) {
+                vgpu_cfg_space(vgpu)[PCI_CLASS_DEVICE] =
+                        INTEL_GVT_PCI_CLASS_VGA_OTHER;
+                vgpu_cfg_space(vgpu)[PCI_CLASS_PROG] =
+                        INTEL_GVT_PCI_CLASS_VGA_OTHER;
+        }
+
+        /* Show guest that there isn't any stolen memory.*/
+        gmch_ctl = (u16 *)(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_GMCH_CONTROL);
+        *gmch_ctl &= ~(BDW_GMCH_GMS_MASK << BDW_GMCH_GMS_SHIFT);
+
+        intel_vgpu_write_pci_bar(vgpu, PCI_BASE_ADDRESS_2,
+                                 gvt_aperture_pa_base(gvt), true);
+
+        vgpu_cfg_space(vgpu)[PCI_COMMAND] &= ~(PCI_COMMAND_IO
+                                             | PCI_COMMAND_MEMORY
+                                             | PCI_COMMAND_MASTER);
+        /*
+         * Clear the bar upper 32bit and let guest to assign the new value
+         */
+        memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4);
+        memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4);
+
+        for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) {
+                vgpu->cfg_space.bar[i].size = pci_resource_len(
+                                              gvt->dev_priv->drm.pdev, i * 2);
+                vgpu->cfg_space.bar[i].tracked = false;
+        }
+}
+
+static void populate_pvinfo_page(struct intel_vgpu *vgpu)
+{
+        /* setup the ballooning information */
+        vgpu_vreg64(vgpu, vgtif_reg(magic)) = VGT_MAGIC;
+        vgpu_vreg(vgpu, vgtif_reg(version_major)) = 1;
+        vgpu_vreg(vgpu, vgtif_reg(version_minor)) = 0;
+        vgpu_vreg(vgpu, vgtif_reg(display_ready)) = 0;
+        vgpu_vreg(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
+        vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
+                vgpu_aperture_gmadr_base(vgpu);
+        vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) =
+                vgpu_aperture_sz(vgpu);
+        vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) =
+                vgpu_hidden_gmadr_base(vgpu);
+        vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) =
+                vgpu_hidden_sz(vgpu);
+
+        vgpu_vreg(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu);
+
+        gvt_dbg_core("Populate PVINFO PAGE for vGPU %d\n", vgpu->id);
+        gvt_dbg_core("aperture base [GMADR] 0x%llx size 0x%llx\n",
+                vgpu_aperture_gmadr_base(vgpu), vgpu_aperture_sz(vgpu));
+        gvt_dbg_core("hidden base [GMADR] 0x%llx size=0x%llx\n",
+                vgpu_hidden_gmadr_base(vgpu), vgpu_hidden_sz(vgpu));
+        gvt_dbg_core("fence size %d\n", vgpu_fence_sz(vgpu));
+
+        WARN_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
+}
+
+/**
+ * intel_gvt_destroy_vgpu - destroy a virtual GPU
+ * @vgpu: virtual GPU
+ *
+ * This function is called when user wants to destroy a virtual GPU.
+ *
+ */
+void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu)
+{
+        struct intel_gvt *gvt = vgpu->gvt;
+
+        mutex_lock(&gvt->lock);
+
+        vgpu->active = false;
+        idr_remove(&gvt->vgpu_idr, vgpu->id);
+
+        if (atomic_read(&vgpu->running_workload_num)) {
+                mutex_unlock(&gvt->lock);
+                intel_gvt_wait_vgpu_idle(vgpu);
+                mutex_lock(&gvt->lock);
+        }
+
+        intel_vgpu_stop_schedule(vgpu);
+        intel_vgpu_clean_sched_policy(vgpu);
+        intel_vgpu_clean_gvt_context(vgpu);
+        intel_vgpu_clean_execlist(vgpu);
+        intel_vgpu_clean_display(vgpu);
+        intel_vgpu_clean_opregion(vgpu);
+        intel_vgpu_clean_gtt(vgpu);
+        intel_gvt_hypervisor_detach_vgpu(vgpu);
+        intel_vgpu_free_resource(vgpu);
+        clean_vgpu_mmio(vgpu);
+        vfree(vgpu);
+
+        mutex_unlock(&gvt->lock);
+}
+
+/**
+ * intel_gvt_create_vgpu - create a virtual GPU
+ * @gvt: GVT device
+ * @param: vGPU creation parameters
+ *
+ * This function is called when user wants to create a virtual GPU.
+ *
+ * Returns:
+ * pointer to intel_vgpu, error pointer if failed.
+ */
+struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
+                struct intel_vgpu_creation_params *param)
+{
+        struct intel_vgpu *vgpu;
+        int ret;
+
+        gvt_dbg_core("handle %llu low %llu MB high %llu MB fence %llu\n",
+                        param->handle, param->low_gm_sz, param->high_gm_sz,
+                        param->fence_sz);
+
+        vgpu = vzalloc(sizeof(*vgpu));
+        if (!vgpu)
+                return ERR_PTR(-ENOMEM);
+
+        mutex_lock(&gvt->lock);
+
+        ret = idr_alloc(&gvt->vgpu_idr, vgpu, 1, GVT_MAX_VGPU, GFP_KERNEL);
+        if (ret < 0)
+                goto out_free_vgpu;
+
+        vgpu->id = ret;
+        vgpu->handle = param->handle;
+        vgpu->gvt = gvt;
+        bitmap_zero(vgpu->tlb_handle_pending, I915_NUM_ENGINES);
+
+        setup_vgpu_cfg_space(vgpu, param);
+
+        ret = setup_vgpu_mmio(vgpu);
+        if (ret)
+                goto out_free_vgpu;
+
+        ret = intel_vgpu_alloc_resource(vgpu, param);
+        if (ret)
+                goto out_clean_vgpu_mmio;
+
+        populate_pvinfo_page(vgpu);
+
+        ret = intel_gvt_hypervisor_attach_vgpu(vgpu);
+        if (ret)
+                goto out_clean_vgpu_resource;
+
+        ret = intel_vgpu_init_gtt(vgpu);
+        if (ret)
+                goto out_detach_hypervisor_vgpu;
+
+        if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) {
+                ret = intel_vgpu_init_opregion(vgpu, 0);
+                if (ret)
+                        goto out_clean_gtt;
+        }
+
+        ret = intel_vgpu_init_display(vgpu);
+        if (ret)
+                goto out_clean_opregion;
+
+        ret = intel_vgpu_init_execlist(vgpu);
+        if (ret)
+                goto out_clean_display;
+
+        ret = intel_vgpu_init_gvt_context(vgpu);
+        if (ret)
+                goto out_clean_execlist;
+
+        ret = intel_vgpu_init_sched_policy(vgpu);
+        if (ret)
+                goto out_clean_shadow_ctx;
+
+        vgpu->active = true;
+        mutex_unlock(&gvt->lock);
+
+        return vgpu;
+
+out_clean_shadow_ctx:
+        intel_vgpu_clean_gvt_context(vgpu);
+out_clean_execlist:
+        intel_vgpu_clean_execlist(vgpu);
+out_clean_display:
+        intel_vgpu_clean_display(vgpu);
+out_clean_opregion:
+        intel_vgpu_clean_opregion(vgpu);
+out_clean_gtt:
+        intel_vgpu_clean_gtt(vgpu);
+out_detach_hypervisor_vgpu:
+        intel_gvt_hypervisor_detach_vgpu(vgpu);
+out_clean_vgpu_resource:
+        intel_vgpu_free_resource(vgpu);
+out_clean_vgpu_mmio:
+        clean_vgpu_mmio(vgpu);
+out_free_vgpu:
+        vfree(vgpu);
+        mutex_unlock(&gvt->lock);
+        return ERR_PTR(ret);
+}
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 70980f82a15b..f191d7b66b1d 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -1308,10 +1308,11 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         bool active = false;
 
         /* If the command parser is not enabled, report 0 - unsupported */
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 if (intel_engine_needs_cmd_parser(engine)) {
                         active = true;
                         break;
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 6c7bb87f764e..20638d22bbad 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -79,10 +79,8 @@ static int i915_capabilities(struct seq_file *m, void *data)
         seq_printf(m, "gen: %d\n", INTEL_GEN(dev_priv));
         seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev_priv));
 #define PRINT_FLAG(x)  seq_printf(m, #x ": %s\n", yesno(info->x))
-#define SEP_SEMICOLON ;
-        DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_SEMICOLON);
+        DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
-#undef SEP_SEMICOLON
 
         return 0;
 }
@@ -109,7 +107,7 @@ static char get_tiling_flag(struct drm_i915_gem_object *obj)
 
 static char get_global_flag(struct drm_i915_gem_object *obj)
 {
-        return i915_gem_object_to_ggtt(obj, NULL) ?  'g' : ' ';
+        return obj->fault_mappable ? 'g' : ' ';
 }
 
 static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
@@ -152,7 +150,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
                    obj->base.size / 1024,
                    obj->base.read_domains,
                    obj->base.write_domain);
-        for_each_engine_id(engine, dev_priv, id)
+        for_each_engine(engine, dev_priv, id)
                 seq_printf(m, "%x ",
                            i915_gem_active_get_seqno(&obj->last_read[id],
                                                      &obj->base.dev->struct_mutex));
@@ -188,15 +186,6 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
         }
         if (obj->stolen)
                 seq_printf(m, " (stolen: %08llx)", obj->stolen->start);
-        if (obj->pin_display || obj->fault_mappable) {
-                char s[3], *t = s;
-                if (obj->pin_display)
-                        *t++ = 'p';
-                if (obj->fault_mappable)
-                        *t++ = 'f';
-                *t = '\0';
-                seq_printf(m, " (%s mappable)", s);
-        }
 
         engine = i915_gem_active_get_engine(&obj->last_write,
                                             &dev_priv->drm.struct_mutex);
@@ -334,11 +323,12 @@ static void print_batch_pool_stats(struct seq_file *m,
         struct drm_i915_gem_object *obj;
         struct file_stats stats;
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int j;
 
         memset(&stats, 0, sizeof(stats));
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 for (j = 0; j < ARRAY_SIZE(engine->batch_pool.cache_list); j++) {
                         list_for_each_entry(obj,
                                             &engine->batch_pool.cache_list[j],
@@ -402,7 +392,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
         if (ret)
                 return ret;
 
-        seq_printf(m, "%u objects, %zu bytes\n",
+        seq_printf(m, "%u objects, %llu bytes\n",
                    dev_priv->mm.object_count,
                    dev_priv->mm.object_memory);
 
@@ -607,6 +597,7 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
         struct drm_device *dev = &dev_priv->drm;
         struct drm_i915_gem_object *obj;
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int total = 0;
         int ret, j;
 
@@ -614,7 +605,7 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
         if (ret)
                 return ret;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 for (j = 0; j < ARRAY_SIZE(engine->batch_pool.cache_list); j++) {
                         int count;
 
@@ -645,12 +636,30 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
         return 0;
 }
 
+static void print_request(struct seq_file *m,
+                          struct drm_i915_gem_request *rq,
+                          const char *prefix)
+{
+        struct pid *pid = rq->ctx->pid;
+        struct task_struct *task;
+
+        rcu_read_lock();
+        task = pid ? pid_task(pid, PIDTYPE_PID) : NULL;
+        seq_printf(m, "%s%x [%x:%x] @ %d: %s [%d]\n", prefix,
+                   rq->fence.seqno, rq->ctx->hw_id, rq->fence.seqno,
+                   jiffies_to_msecs(jiffies - rq->emitted_jiffies),
+                   task ? task->comm : "<unknown>",
+                   task ? task->pid : -1);
+        rcu_read_unlock();
+}
+
 static int i915_gem_request_info(struct seq_file *m, void *data)
 {
         struct drm_i915_private *dev_priv = node_to_i915(m->private);
         struct drm_device *dev = &dev_priv->drm;
-        struct intel_engine_cs *engine;
         struct drm_i915_gem_request *req;
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int ret, any;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -658,7 +667,7 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
                 return ret;
 
         any = 0;
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 int count;
 
                 count = 0;
@@ -668,19 +677,8 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
                         continue;
 
                 seq_printf(m, "%s requests: %d\n", engine->name, count);
-                list_for_each_entry(req, &engine->request_list, link) {
-                        struct pid *pid = req->ctx->pid;
-                        struct task_struct *task;
-
-                        rcu_read_lock();
-                        task = pid ? pid_task(pid, PIDTYPE_PID) : NULL;
-                        seq_printf(m, "    %x @ %d: %s [%d]\n",
-                                   req->fence.seqno,
-                                   (int) (jiffies - req->emitted_jiffies),
-                                   task ? task->comm : "<unknown>",
-                                   task ? task->pid : -1);
-                        rcu_read_unlock();
-                }
+                list_for_each_entry(req, &engine->request_list, link)
+                        print_request(m, req, "    ");
 
                 any++;
         }
@@ -715,8 +713,9 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
         struct drm_i915_private *dev_priv = node_to_i915(m->private);
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 i915_ring_seqno_info(m, engine);
 
         return 0;
@@ -727,6 +726,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 {
         struct drm_i915_private *dev_priv = node_to_i915(m->private);
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int i, pipe;
 
         intel_runtime_pm_get(dev_priv);
@@ -895,7 +895,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                 seq_printf(m, "Graphics Interrupt mask:         %08x\n",
                            I915_READ(GTIMR));
         }
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 if (INTEL_GEN(dev_priv) >= 6) {
                         seq_printf(m,
                                    "Graphics Interrupt mask (%s):       %08x\n",
@@ -943,7 +943,7 @@ static int i915_hws_info(struct seq_file *m, void *data)
         const u32 *hws;
         int i;
 
-        engine = &dev_priv->engine[(uintptr_t)node->info_ent->data];
+        engine = dev_priv->engine[(uintptr_t)node->info_ent->data];
         hws = engine->status_page.page_addr;
         if (hws == NULL)
                 return 0;
@@ -956,6 +956,8 @@ static int i915_hws_info(struct seq_file *m, void *data)
         return 0;
 }
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 static ssize_t
 i915_error_state_write(struct file *filp,
                        const char __user *ubuf,
@@ -1038,6 +1040,8 @@ static const struct file_operations i915_error_state_fops = {
         .release = i915_error_state_release,
 };
 
+#endif
+
 static int
 i915_next_seqno_get(void *data, u64 *val)
 {
@@ -1277,15 +1281,42 @@ out:
         return ret;
 }
 
+static void i915_instdone_info(struct drm_i915_private *dev_priv,
+                               struct seq_file *m,
+                               struct intel_instdone *instdone)
+{
+        int slice;
+        int subslice;
+
+        seq_printf(m, "\t\tINSTDONE: 0x%08x\n",
+                   instdone->instdone);
+
+        if (INTEL_GEN(dev_priv) <= 3)
+                return;
+
+        seq_printf(m, "\t\tSC_INSTDONE: 0x%08x\n",
+                   instdone->slice_common);
+
+        if (INTEL_GEN(dev_priv) <= 6)
+                return;
+
+        for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+                seq_printf(m, "\t\tSAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
+                           slice, subslice, instdone->sampler[slice][subslice]);
+
+        for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+                seq_printf(m, "\t\tROW_INSTDONE[%d][%d]: 0x%08x\n",
+                           slice, subslice, instdone->row[slice][subslice]);
+}
+
 static int i915_hangcheck_info(struct seq_file *m, void *unused)
 {
         struct drm_i915_private *dev_priv = node_to_i915(m->private);
         struct intel_engine_cs *engine;
         u64 acthd[I915_NUM_ENGINES];
         u32 seqno[I915_NUM_ENGINES];
-        u32 instdone[I915_NUM_INSTDONE_REG];
+        struct intel_instdone instdone;
         enum intel_engine_id id;
-        int j;
 
         if (test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
                 seq_printf(m, "Wedged\n");
@@ -1303,12 +1334,12 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 
         intel_runtime_pm_get(dev_priv);
 
-        for_each_engine_id(engine, dev_priv, id) {
+        for_each_engine(engine, dev_priv, id) {
                 acthd[id] = intel_engine_get_active_head(engine);
                 seqno[id] = intel_engine_get_seqno(engine);
         }
 
-        i915_get_extra_instdone(dev_priv, instdone);
+        intel_engine_get_instdone(dev_priv->engine[RCS], &instdone);
 
         intel_runtime_pm_put(dev_priv);
 
@@ -1319,7 +1350,10 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
         } else
                 seq_printf(m, "Hangcheck inactive\n");
 
-        for_each_engine_id(engine, dev_priv, id) {
+        for_each_engine(engine, dev_priv, id) {
+                struct intel_breadcrumbs *b = &engine->breadcrumbs;
+                struct rb_node *rb;
+
                 seq_printf(m, "%s:\n", engine->name);
                 seq_printf(m, "\tseqno = %x [current %x, last %x]\n",
                            engine->hangcheck.seqno,
@@ -1329,6 +1363,15 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
                            yesno(intel_engine_has_waiter(engine)),
                            yesno(test_bit(engine->id,
                                           &dev_priv->gpu_error.missed_irq_rings)));
+                spin_lock(&b->lock);
+                for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
+                        struct intel_wait *w = container_of(rb, typeof(*w), node);
+
+                        seq_printf(m, "\t%s [%d] waiting for %x\n",
+                                   w->tsk->comm, w->tsk->pid, w->seqno);
+                }
+                spin_unlock(&b->lock);
+
                 seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n",
                            (long long)engine->hangcheck.acthd,
                            (long long)acthd[id]);
@@ -1336,18 +1379,14 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
                 seq_printf(m, "\taction = %d\n", engine->hangcheck.action);
 
                 if (engine->id == RCS) {
-                        seq_puts(m, "\tinstdone read =");
-
-                        for (j = 0; j < I915_NUM_INSTDONE_REG; j++)
-                                seq_printf(m, " 0x%08x", instdone[j]);
+                        seq_puts(m, "\tinstdone read =\n");
 
-                        seq_puts(m, "\n\tinstdone accu =");
+                        i915_instdone_info(dev_priv, m, &instdone);
 
-                        for (j = 0; j < I915_NUM_INSTDONE_REG; j++)
-                                seq_printf(m, " 0x%08x",
-                                           engine->hangcheck.instdone[j]);
+                        seq_puts(m, "\tinstdone accu =\n");
 
-                        seq_puts(m, "\n");
+                        i915_instdone_info(dev_priv, m,
+                                           &engine->hangcheck.instdone);
                 }
         }
 
@@ -1635,7 +1674,8 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
                 seq_printf(m, "FBC disabled: %s\n",
                            dev_priv->fbc.no_fbc_reason);
 
-        if (INTEL_GEN(dev_priv) >= 7)
+        if (intel_fbc_is_active(dev_priv) &&
+            INTEL_GEN(dev_priv) >= 7)
                 seq_printf(m, "Compressing: %s\n",
                            yesno(I915_READ(FBC_STATUS2) &
                                  FBC_COMPRESSION_MASK));
@@ -1909,6 +1949,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
         struct drm_device *dev = &dev_priv->drm;
         struct intel_engine_cs *engine;
         struct i915_gem_context *ctx;
+        enum intel_engine_id id;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1935,7 +1976,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
                 seq_putc(m, ctx->remap_slice ? 'R' : 'r');
                 seq_putc(m, '\n');
 
-                for_each_engine(engine, dev_priv) {
+                for_each_engine(engine, dev_priv, id) {
                         struct intel_context *ce = &ctx->engine[engine->id];
 
                         seq_printf(m, "%s: ", engine->name);
@@ -2002,6 +2043,7 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
         struct drm_device *dev = &dev_priv->drm;
         struct intel_engine_cs *engine;
         struct i915_gem_context *ctx;
+        enum intel_engine_id id;
         int ret;
 
         if (!i915.enable_execlists) {
@@ -2014,7 +2056,7 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
                 return ret;
 
         list_for_each_entry(ctx, &dev_priv->context_list, link)
-                for_each_engine(engine, dev_priv)
+                for_each_engine(engine, dev_priv, id)
                         i915_dump_lrc_obj(m, ctx, engine);
 
         mutex_unlock(&dev->struct_mutex);
@@ -2022,84 +2064,6 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
         return 0;
 }
 
-static int i915_execlists(struct seq_file *m, void *data)
-{
-        struct drm_i915_private *dev_priv = node_to_i915(m->private);
-        struct drm_device *dev = &dev_priv->drm;
-        struct intel_engine_cs *engine;
-        u32 status_pointer;
-        u8 read_pointer;
-        u8 write_pointer;
-        u32 status;
-        u32 ctx_id;
-        struct list_head *cursor;
-        int i, ret;
-
-        if (!i915.enable_execlists) {
-                seq_puts(m, "Logical Ring Contexts are disabled\n");
-                return 0;
-        }
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
-
-        intel_runtime_pm_get(dev_priv);
-
-        for_each_engine(engine, dev_priv) {
-                struct drm_i915_gem_request *head_req = NULL;
-                int count = 0;
-
-                seq_printf(m, "%s\n", engine->name);
-
-                status = I915_READ(RING_EXECLIST_STATUS_LO(engine));
-                ctx_id = I915_READ(RING_EXECLIST_STATUS_HI(engine));
-                seq_printf(m, "\tExeclist status: 0x%08X, context: %u\n",
-                           status, ctx_id);
-
-                status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
-                seq_printf(m, "\tStatus pointer: 0x%08X\n", status_pointer);
-
-                read_pointer = GEN8_CSB_READ_PTR(status_pointer);
-                write_pointer = GEN8_CSB_WRITE_PTR(status_pointer);
-                if (read_pointer > write_pointer)
-                        write_pointer += GEN8_CSB_ENTRIES;
-                seq_printf(m, "\tRead pointer: 0x%08X, write pointer 0x%08X\n",
-                           read_pointer, write_pointer);
-
-                for (i = 0; i < GEN8_CSB_ENTRIES; i++) {
-                        status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, i));
-                        ctx_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, i));
-
-                        seq_printf(m, "\tStatus buffer %d: 0x%08X, context: %u\n",
-                                   i, status, ctx_id);
-                }
-
-                spin_lock_bh(&engine->execlist_lock);
-                list_for_each(cursor, &engine->execlist_queue)
-                        count++;
-                head_req = list_first_entry_or_null(&engine->execlist_queue,
-                                                    struct drm_i915_gem_request,
-                                                    execlist_link);
-                spin_unlock_bh(&engine->execlist_lock);
-
-                seq_printf(m, "\t%d requests in queue\n", count);
-                if (head_req) {
-                        seq_printf(m, "\tHead request context: %u\n",
-                                   head_req->ctx->hw_id);
-                        seq_printf(m, "\tHead request tail: %u\n",
-                                   head_req->tail);
-                }
-
-                seq_putc(m, '\n');
-        }
-
-        intel_runtime_pm_put(dev_priv);
-        mutex_unlock(&dev->struct_mutex);
-
-        return 0;
-}
-
 static const char *swizzle_string(unsigned swizzle)
 {
         switch (swizzle) {
@@ -2201,14 +2165,15 @@ static int per_file_ctx(int id, void *ptr, void *data)
 static void gen8_ppgtt_info(struct seq_file *m,
                             struct drm_i915_private *dev_priv)
 {
-        struct intel_engine_cs *engine;
         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int i;
 
         if (!ppgtt)
                 return;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 seq_printf(m, "%s\n", engine->name);
                 for (i = 0; i < 4; i++) {
                         u64 pdp = I915_READ(GEN8_RING_PDP_UDW(engine, i));
@@ -2223,11 +2188,12 @@ static void gen6_ppgtt_info(struct seq_file *m,
                             struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
         if (IS_GEN6(dev_priv))
                 seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE));
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 seq_printf(m, "%s\n", engine->name);
                 if (IS_GEN7(dev_priv))
                         seq_printf(m, "GFX_MODE: 0x%08x\n",
@@ -2296,9 +2262,10 @@ out_unlock:
 static int count_irq_waiters(struct drm_i915_private *i915)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int count = 0;
 
-        for_each_engine(engine, i915)
+        for_each_engine(engine, i915, id)
                 count += intel_engine_has_waiter(engine);
 
         return count;
@@ -2461,7 +2428,7 @@ static void i915_guc_client_info(struct seq_file *m,
         seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail);
         seq_printf(m, "\tLast submission result: %d\n", client->retcode);
 
-        for_each_engine_id(engine, dev_priv, id) {
+        for_each_engine(engine, dev_priv, id) {
                 u64 submissions = client->submissions[id];
                 tot += submissions;
                 seq_printf(m, "\tSubmissions: %llu %s\n",
@@ -2504,7 +2471,7 @@ static int i915_guc_info(struct seq_file *m, void *data)
         seq_printf(m, "GuC last action error code: %d\n", guc.action_err);
 
         seq_printf(m, "\nGuC submissions:\n");
-        for_each_engine_id(engine, dev_priv, id) {
+        for_each_engine(engine, dev_priv, id) {
                 u64 submissions = guc.submissions[id];
                 total += submissions;
                 seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x\n",
@@ -3121,6 +3088,134 @@ static int i915_display_info(struct seq_file *m, void *unused)
         return 0;
 }
 
+static int i915_engine_info(struct seq_file *m, void *unused)
+{
+        struct drm_i915_private *dev_priv = node_to_i915(m->private);
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
+
+        for_each_engine(engine, dev_priv, id) {
+                struct intel_breadcrumbs *b = &engine->breadcrumbs;
+                struct drm_i915_gem_request *rq;
+                struct rb_node *rb;
+                u64 addr;
+
+                seq_printf(m, "%s\n", engine->name);
+                seq_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [score %d]\n",
+                           intel_engine_get_seqno(engine),
+                           engine->last_submitted_seqno,
+                           engine->hangcheck.seqno,
+                           engine->hangcheck.score);
+
+                rcu_read_lock();
+
+                seq_printf(m, "\tRequests:\n");
+
+                rq = list_first_entry(&engine->request_list,
+                                struct drm_i915_gem_request, link);
+                if (&rq->link != &engine->request_list)
+                        print_request(m, rq, "\t\tfirst  ");
+
+                rq = list_last_entry(&engine->request_list,
+                                struct drm_i915_gem_request, link);
+                if (&rq->link != &engine->request_list)
+                        print_request(m, rq, "\t\tlast   ");
+
+                rq = i915_gem_find_active_request(engine);
+                if (rq) {
+                        print_request(m, rq, "\t\tactive ");
+                        seq_printf(m,
+                                   "\t\t[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]\n",
+                                   rq->head, rq->postfix, rq->tail,
+                                   rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
+                                   rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
+                }
+
+                seq_printf(m, "\tRING_START: 0x%08x [0x%08x]\n",
+                           I915_READ(RING_START(engine->mmio_base)),
+                           rq ? i915_ggtt_offset(rq->ring->vma) : 0);
+                seq_printf(m, "\tRING_HEAD:  0x%08x [0x%08x]\n",
+                           I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR,
+                           rq ? rq->ring->head : 0);
+                seq_printf(m, "\tRING_TAIL:  0x%08x [0x%08x]\n",
+                           I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
+                           rq ? rq->ring->tail : 0);
+                seq_printf(m, "\tRING_CTL:   0x%08x [%s]\n",
+                           I915_READ(RING_CTL(engine->mmio_base)),
+                           I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : "");
+
+                rcu_read_unlock();
+
+                addr = intel_engine_get_active_head(engine);
+                seq_printf(m, "\tACTHD:  0x%08x_%08x\n",
+                           upper_32_bits(addr), lower_32_bits(addr));
+                addr = intel_engine_get_last_batch_head(engine);
+                seq_printf(m, "\tBBADDR: 0x%08x_%08x\n",
+                           upper_32_bits(addr), lower_32_bits(addr));
+
+                if (i915.enable_execlists) {
+                        u32 ptr, read, write;
+
+                        seq_printf(m, "\tExeclist status: 0x%08x %08x\n",
+                                   I915_READ(RING_EXECLIST_STATUS_LO(engine)),
+                                   I915_READ(RING_EXECLIST_STATUS_HI(engine)));
+
+                        ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
+                        read = GEN8_CSB_READ_PTR(ptr);
+                        write = GEN8_CSB_WRITE_PTR(ptr);
+                        seq_printf(m, "\tExeclist CSB read %d, write %d\n",
+                                   read, write);
+                        if (read >= GEN8_CSB_ENTRIES)
+                                read = 0;
+                        if (write >= GEN8_CSB_ENTRIES)
+                                write = 0;
+                        if (read > write)
+                                write += GEN8_CSB_ENTRIES;
+                        while (read < write) {
+                                unsigned int idx = ++read % GEN8_CSB_ENTRIES;
+
+                                seq_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
+                                           idx,
+                                           I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
+                                           I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)));
+                        }
+
+                        rcu_read_lock();
+                        rq = READ_ONCE(engine->execlist_port[0].request);
+                        if (rq)
+                                print_request(m, rq, "\t\tELSP[0] ");
+                        else
+                                seq_printf(m, "\t\tELSP[0] idle\n");
+                        rq = READ_ONCE(engine->execlist_port[1].request);
+                        if (rq)
+                                print_request(m, rq, "\t\tELSP[1] ");
+                        else
+                                seq_printf(m, "\t\tELSP[1] idle\n");
+                        rcu_read_unlock();
+                } else if (INTEL_GEN(dev_priv) > 6) {
+                        seq_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
+                                   I915_READ(RING_PP_DIR_BASE(engine)));
+                        seq_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
+                                   I915_READ(RING_PP_DIR_BASE_READ(engine)));
+                        seq_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
+                                   I915_READ(RING_PP_DIR_DCLV(engine)));
+                }
+
+                spin_lock(&b->lock);
+                for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
+                        struct intel_wait *w = container_of(rb, typeof(*w), node);
+
+                        seq_printf(m, "\t%s [%d] waiting for %x\n",
+                                   w->tsk->comm, w->tsk->pid, w->seqno);
+                }
+                spin_unlock(&b->lock);
+
+                seq_puts(m, "\n");
+        }
+
+        return 0;
+}
+
 static int i915_semaphore_status(struct seq_file *m, void *unused)
 {
         struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -3147,7 +3242,7 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
                 page = i915_gem_object_get_page(dev_priv->semaphore->obj, 0);
 
                 seqno = (uint64_t *)kmap_atomic(page);
-                for_each_engine_id(engine, dev_priv, id) {
+                for_each_engine(engine, dev_priv, id) {
                         uint64_t offset;
 
                         seq_printf(m, "%s\n", engine->name);
@@ -3172,7 +3267,7 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
                 kunmap_atomic(seqno);
         } else {
                 seq_puts(m, "  Last signal:");
-                for_each_engine(engine, dev_priv)
+                for_each_engine(engine, dev_priv, id)
                         for (j = 0; j < num_rings; j++)
                                 seq_printf(m, "0x%08x\n",
                                            I915_READ(engine->semaphore.mbox.signal[j]));
@@ -3180,7 +3275,7 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
         }
 
         seq_puts(m, "\nSync seqno:\n");
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 for (j = 0; j < num_rings; j++)
                         seq_printf(m, "  0x%08x ",
                                    engine->semaphore.sync_seqno[j]);
@@ -3236,7 +3331,7 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
         intel_runtime_pm_get(dev_priv);
 
         seq_printf(m, "Workarounds applied: %d\n", workarounds->count);
-        for_each_engine_id(engine, dev_priv, id)
+        for_each_engine(engine, dev_priv, id)
                 seq_printf(m, "HW whitelist count for %s: %d\n",
                            engine->name, workarounds->hw_whitelist_count[id]);
         for (i = 0; i < workarounds->count; ++i) {
@@ -4462,7 +4557,7 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
         else if (IS_VALLEYVIEW(dev_priv))
                 num_levels = 1;
         else
-                num_levels = ilk_wm_max_level(dev) + 1;
+                num_levels = ilk_wm_max_level(dev_priv) + 1;
 
         drm_modeset_lock_all(dev);
 
@@ -4578,7 +4673,7 @@ static ssize_t wm_latency_write(struct file *file, const char __user *ubuf,
         else if (IS_VALLEYVIEW(dev_priv))
                 num_levels = 1;
         else
-                num_levels = ilk_wm_max_level(dev) + 1;
+                num_levels = ilk_wm_max_level(dev_priv) + 1;
 
         if (len >= sizeof(tmp))
                 return -EINVAL;
@@ -5274,7 +5369,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
         {"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
         {"i915_context_status", i915_context_status, 0},
         {"i915_dump_lrc", i915_dump_lrc, 0},
-        {"i915_execlists", i915_execlists, 0},
         {"i915_forcewake_domains", i915_forcewake_domains, 0},
         {"i915_swizzle_info", i915_swizzle_info, 0},
         {"i915_ppgtt_info", i915_ppgtt_info, 0},
@@ -5286,6 +5380,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
         {"i915_power_domain_info", i915_power_domain_info, 0},
         {"i915_dmc_info", i915_dmc_info, 0},
         {"i915_display_info", i915_display_info, 0},
+        {"i915_engine_info", i915_engine_info, 0},
         {"i915_semaphore_status", i915_semaphore_status, 0},
         {"i915_shared_dplls_info", i915_shared_dplls_info, 0},
         {"i915_dp_mst_info", i915_dp_mst_info, 0},
@@ -5308,7 +5403,9 @@ static const struct i915_debugfs_files {
         {"i915_ring_missed_irq", &i915_ring_missed_irq_fops},
         {"i915_ring_test_irq", &i915_ring_test_irq_fops},
         {"i915_gem_drop_caches", &i915_drop_caches_fops},
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
         {"i915_error_state", &i915_error_state_fops},
+#endif
         {"i915_next_seqno", &i915_next_seqno_fops},
         {"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
         {"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 7f4e8adec8a8..912d5348e3e7 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -114,7 +114,7 @@ static bool i915_error_injected(struct drm_i915_private *dev_priv)
                       fmt, ##__VA_ARGS__)
 
 
-static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
+static enum intel_pch intel_virt_detect_pch(struct drm_i915_private *dev_priv)
 {
         enum intel_pch ret = PCH_NOP;
 
@@ -125,16 +125,16 @@ static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
          * make an educated guess as to which PCH is really there.
          */
 
-        if (IS_GEN5(dev)) {
+        if (IS_GEN5(dev_priv)) {
                 ret = PCH_IBX;
                 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
-        } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
+        } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
                 ret = PCH_CPT;
                 DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
-        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 ret = PCH_LPT;
                 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
-        } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 ret = PCH_SPT;
                 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
         }
@@ -174,40 +174,46 @@ static void intel_detect_pch(struct drm_device *dev)
                         if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_IBX;
                                 DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
-                                WARN_ON(!IS_GEN5(dev));
+                                WARN_ON(!IS_GEN5(dev_priv));
                         } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_CPT;
                                 DRM_DEBUG_KMS("Found CougarPoint PCH\n");
-                                WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
+                                WARN_ON(!(IS_GEN6(dev_priv) ||
+                                        IS_IVYBRIDGE(dev_priv)));
                         } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
                                 /* PantherPoint is CPT compatible */
                                 dev_priv->pch_type = PCH_CPT;
                                 DRM_DEBUG_KMS("Found PantherPoint PCH\n");
-                                WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
+                                WARN_ON(!(IS_GEN6(dev_priv) ||
+                                        IS_IVYBRIDGE(dev_priv)));
                         } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_LPT;
                                 DRM_DEBUG_KMS("Found LynxPoint PCH\n");
-                                WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
-                                WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
+                                WARN_ON(!IS_HASWELL(dev_priv) &&
+                                        !IS_BROADWELL(dev_priv));
+                                WARN_ON(IS_HSW_ULT(dev_priv) ||
+                                        IS_BDW_ULT(dev_priv));
                         } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_LPT;
                                 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
-                                WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
-                                WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
+                                WARN_ON(!IS_HASWELL(dev_priv) &&
+                                        !IS_BROADWELL(dev_priv));
+                                WARN_ON(!IS_HSW_ULT(dev_priv) &&
+                                        !IS_BDW_ULT(dev_priv));
                         } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_SPT;
                                 DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
-                                WARN_ON(!IS_SKYLAKE(dev) &&
-                                        !IS_KABYLAKE(dev));
+                                WARN_ON(!IS_SKYLAKE(dev_priv) &&
+                                        !IS_KABYLAKE(dev_priv));
                         } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_SPT;
                                 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
-                                WARN_ON(!IS_SKYLAKE(dev) &&
-                                        !IS_KABYLAKE(dev));
+                                WARN_ON(!IS_SKYLAKE(dev_priv) &&
+                                        !IS_KABYLAKE(dev_priv));
                         } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_KBP;
                                 DRM_DEBUG_KMS("Found KabyPoint PCH\n");
-                                WARN_ON(!IS_KABYLAKE(dev));
+                                WARN_ON(!IS_KABYLAKE(dev_priv));
                         } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
                                    (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
                                    ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
@@ -215,7 +221,8 @@ static void intel_detect_pch(struct drm_device *dev)
                                             PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
                                     pch->subsystem_device ==
                                             PCI_SUBDEVICE_ID_QEMU)) {
-                                dev_priv->pch_type = intel_virt_detect_pch(dev);
+                                dev_priv->pch_type =
+                                        intel_virt_detect_pch(dev_priv);
                         } else
                                 continue;
 
@@ -255,16 +262,16 @@ static int i915_getparam(struct drm_device *dev, void *data,
                 value = dev_priv->overlay ? 1 : 0;
                 break;
         case I915_PARAM_HAS_BSD:
-                value = intel_engine_initialized(&dev_priv->engine[VCS]);
+                value = !!dev_priv->engine[VCS];
                 break;
         case I915_PARAM_HAS_BLT:
-                value = intel_engine_initialized(&dev_priv->engine[BCS]);
+                value = !!dev_priv->engine[BCS];
                 break;
         case I915_PARAM_HAS_VEBOX:
-                value = intel_engine_initialized(&dev_priv->engine[VECS]);
+                value = !!dev_priv->engine[VECS];
                 break;
         case I915_PARAM_HAS_BSD2:
-                value = intel_engine_initialized(&dev_priv->engine[VCS2]);
+                value = !!dev_priv->engine[VCS2];
                 break;
         case I915_PARAM_HAS_EXEC_CONSTANTS:
                 value = INTEL_GEN(dev_priv) >= 4;
@@ -417,12 +424,12 @@ intel_setup_mchbar(struct drm_device *dev)
         u32 temp;
         bool enabled;
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 return;
 
         dev_priv->mchbar_need_disable = false;
 
-        if (IS_I915G(dev) || IS_I915GM(dev)) {
+        if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
                 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
                 enabled = !!(temp & DEVEN_MCHBAR_EN);
         } else {
@@ -440,7 +447,7 @@ intel_setup_mchbar(struct drm_device *dev)
         dev_priv->mchbar_need_disable = true;
 
         /* Space is allocated or reserved, so enable it. */
-        if (IS_I915G(dev) || IS_I915GM(dev)) {
+        if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
                 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
                                        temp | DEVEN_MCHBAR_EN);
         } else {
@@ -456,7 +463,7 @@ intel_teardown_mchbar(struct drm_device *dev)
         int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 
         if (dev_priv->mchbar_need_disable) {
-                if (IS_I915G(dev) || IS_I915GM(dev)) {
+                if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
                         u32 deven_val;
 
                         pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
@@ -532,32 +539,6 @@ static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
 
 static void i915_gem_fini(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
-
-        /*
-         * Neither the BIOS, ourselves or any other kernel
-         * expects the system to be in execlists mode on startup,
-         * so we need to reset the GPU back to legacy mode. And the only
-         * known way to disable logical contexts is through a GPU reset.
-         *
-         * So in order to leave the system in a known default configuration,
-         * always reset the GPU upon unload. Afterwards we then clean up the
-         * GEM state tracking, flushing off the requests and leaving the
-         * system in a known idle state.
-         *
-         * Note that is of the upmost importance that the GPU is idle and
-         * all stray writes are flushed *before* we dismantle the backing
-         * storage for the pinned objects.
-         *
-         * However, since we are uncertain that reseting the GPU on older
-         * machines is a good idea, we don't - just in case it leaves the
-         * machine in an unusable condition.
-         */
-        if (HAS_HW_CONTEXTS(dev)) {
-                int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
-                WARN_ON(reset && reset != -ENODEV);
-        }
-
         mutex_lock(&dev->struct_mutex);
         i915_gem_cleanup_engines(dev);
         i915_gem_context_fini(dev);
@@ -636,6 +617,8 @@ static int i915_load_modeset_init(struct drm_device *dev)
         return 0;
 
 cleanup_gem:
+        if (i915_gem_suspend(dev))
+                DRM_ERROR("failed to idle hardware; continuing to unload!\n");
         i915_gem_fini(dev);
 cleanup_irq:
         intel_guc_fini(dev);
@@ -771,6 +754,19 @@ static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
         destroy_workqueue(dev_priv->wq);
 }
 
+/*
+ * We don't keep the workarounds for pre-production hardware, so we expect our
+ * driver to fail on these machines in one way or another. A little warning on
+ * dmesg may help both the user and the bug triagers.
+ */
+static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
+{
+        if (IS_HSW_EARLY_SDV(dev_priv) ||
+            IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0))
+                DRM_ERROR("This is a pre-production stepping. "
+                          "It may not be fully functional.\n");
+}
+
 /**
  * i915_driver_init_early - setup state not requiring device access
  * @dev_priv: device private
@@ -838,13 +834,7 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 
         intel_device_info_dump(dev_priv);
 
-        /* Not all pre-production machines fall into this category, only the
-         * very first ones. Almost everything should work, except for maybe
-         * suspend/resume. And we don't implement workarounds that affect only
-         * pre-production machines. */
-        if (IS_HSW_EARLY_SDV(dev_priv))
-                DRM_INFO("This is an early pre-production Haswell machine. "
-                         "It may not be fully functional.\n");
+        intel_detect_preproduction_hw(dev_priv);
 
         return 0;
 
@@ -870,7 +860,7 @@ static int i915_mmio_setup(struct drm_device *dev)
         int mmio_bar;
         int mmio_size;
 
-        mmio_bar = IS_GEN2(dev) ? 1 : 0;
+        mmio_bar = IS_GEN2(dev_priv) ? 1 : 0;
         /*
          * Before gen4, the registers and the GTT are behind different BARs.
          * However, from gen4 onwards, the registers and the GTT are shared
@@ -1023,7 +1013,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
         pci_set_master(pdev);
 
         /* overlay on gen2 is broken and can't address above 1G */
-        if (IS_GEN2(dev)) {
+        if (IS_GEN2(dev_priv)) {
                 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
                 if (ret) {
                         DRM_ERROR("failed to set DMA mask\n");
@@ -1070,7 +1060,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
          * be lost or delayed, but we use them anyways to avoid
          * stuck interrupts on some machines.
          */
-        if (!IS_I945G(dev) && !IS_I945GM(dev)) {
+        if (!IS_I945G(dev_priv) && !IS_I945GM(dev_priv)) {
                 if (pci_enable_msi(pdev) < 0)
                         DRM_DEBUG_DRIVER("can't enable MSI");
         }
@@ -1242,6 +1232,10 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
         DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
                  driver.name, driver.major, driver.minor, driver.patchlevel,
                  driver.date, pci_name(pdev), dev_priv->drm.primary->index);
+        if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+                DRM_INFO("DRM_I915_DEBUG enabled\n");
+        if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
+                DRM_INFO("DRM_I915_DEBUG_GEM enabled\n");
 
         intel_runtime_pm_put(dev_priv);
 
@@ -1447,8 +1441,6 @@ static int i915_drm_suspend(struct drm_device *dev)
 
         dev_priv->suspend_count++;
 
-        intel_display_set_init_power(dev_priv, false);
-
         intel_csr_ucode_suspend(dev_priv);
 
 out:
@@ -1466,6 +1458,8 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
 
         disable_rpm_wakeref_asserts(dev_priv);
 
+        intel_display_set_init_power(dev_priv, false);
+
         fw_csr = !IS_BROXTON(dev_priv) &&
                 suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
         /*
@@ -1721,6 +1715,22 @@ int i915_resume_switcheroo(struct drm_device *dev)
         return i915_drm_resume(dev);
 }
 
+static void disable_engines_irq(struct drm_i915_private *dev_priv)
+{
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
+
+        /* Ensure irq handler finishes, and not run again. */
+        disable_irq(dev_priv->drm.irq);
+        for_each_engine(engine, dev_priv, id)
+                tasklet_kill(&engine->irq_tasklet);
+}
+
+static void enable_engines_irq(struct drm_i915_private *dev_priv)
+{
+        enable_irq(dev_priv->drm.irq);
+}
+
 /**
  * i915_reset - reset chip after a hang
  * @dev: drm device to reset
@@ -1754,7 +1764,11 @@ void i915_reset(struct drm_i915_private *dev_priv)
         error->reset_count++;
 
         pr_notice("drm/i915: Resetting chip after gpu hang\n");
+
+        disable_engines_irq(dev_priv);
         ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
+        enable_engines_irq(dev_priv);
+
         if (ret) {
                 if (ret != -ENODEV)
                         DRM_ERROR("Failed to reset chip: %i\n", ret);
@@ -1786,15 +1800,6 @@ void i915_reset(struct drm_i915_private *dev_priv)
                 goto error;
         }
 
-        /*
-         * rps/rc6 re-init is necessary to restore state lost after the
-         * reset and the re-install of gt irqs. Skip for ironlake per
-         * previous concerns that it doesn't respond well to some forms
-         * of re-init after reset.
-         */
-        intel_sanitize_gt_powersave(dev_priv);
-        intel_autoenable_gt_powersave(dev_priv);
-
 wakeup:
         wake_up_bit(&error->flags, I915_RESET_IN_PROGRESS);
         return;
@@ -1872,7 +1877,17 @@ static int i915_pm_resume(struct device *kdev)
 /* freeze: before creating the hibernation_image */
 static int i915_pm_freeze(struct device *kdev)
 {
-        return i915_pm_suspend(kdev);
+        int ret;
+
+        ret = i915_pm_suspend(kdev);
+        if (ret)
+                return ret;
+
+        ret = i915_gem_freeze(kdev_to_i915(kdev));
+        if (ret)
+                return ret;
+
+        return 0;
 }
 
 static int i915_pm_freeze_late(struct device *kdev)
@@ -2281,7 +2296,7 @@ static int intel_runtime_suspend(struct device *kdev)
         if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
                 return -ENODEV;
 
-        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
+        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
                 return -ENODEV;
 
         DRM_DEBUG_KMS("Suspending device\n");
@@ -2385,7 +2400,7 @@ static int intel_runtime_resume(struct device *kdev)
         struct drm_i915_private *dev_priv = to_i915(dev);
         int ret = 0;
 
-        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
+        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
                 return -ENODEV;
 
         DRM_DEBUG_KMS("Resuming device\n");
@@ -2403,7 +2418,7 @@ static int intel_runtime_resume(struct device *kdev)
         if (IS_GEN6(dev_priv))
                 intel_init_pch_refclk(dev);
 
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 bxt_disable_dc9(dev_priv);
                 bxt_display_core_init(dev_priv, true);
                 if (dev_priv->csr.dmc_payload &&
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 4dd307ed4336..f022f438e5b9 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -70,7 +70,8 @@
 
 #define DRIVER_NAME             "i915"
 #define DRIVER_DESC             "Intel Graphics"
-#define DRIVER_DATE             "20160919"
+#define DRIVER_DATE             "20161024"
+#define DRIVER_TIMESTAMP        1477290335
 
 #undef WARN_ON
 /* Many gcc seem to no see through this and fall over :( */
@@ -185,6 +186,7 @@ enum plane {
 #define sprite_name(p, s) ((p) * INTEL_INFO(dev)->num_sprites[(p)] + (s) + 'A')
 
 enum port {
+        PORT_NONE = -1,
         PORT_A = 0,
         PORT_B,
         PORT_C,
@@ -581,13 +583,25 @@ struct intel_uncore_funcs {
                                 uint32_t val, bool trace);
 };
 
+struct intel_forcewake_range {
+        u32 start;
+        u32 end;
+
+        enum forcewake_domains domains;
+};
+
 struct intel_uncore {
         spinlock_t lock; /** lock is also taken in irq contexts. */
 
+        const struct intel_forcewake_range *fw_domains_table;
+        unsigned int fw_domains_table_entries;
+
         struct intel_uncore_funcs funcs;
 
         unsigned fifo_count;
+
         enum forcewake_domains fw_domains;
+        enum forcewake_domains fw_domains_active;
 
         struct intel_uncore_forcewake_domain {
                 struct drm_i915_private *i915;
@@ -633,54 +647,53 @@ struct intel_csr {
         uint32_t allowed_dc_mask;
 };
 
-#define DEV_INFO_FOR_EACH_FLAG(func, sep) \
-        func(is_mobile) sep \
-        func(is_i85x) sep \
-        func(is_i915g) sep \
-        func(is_i945gm) sep \
-        func(is_g33) sep \
-        func(hws_needs_physical) sep \
-        func(is_g4x) sep \
-        func(is_pineview) sep \
-        func(is_broadwater) sep \
-        func(is_crestline) sep \
-        func(is_ivybridge) sep \
-        func(is_valleyview) sep \
-        func(is_cherryview) sep \
-        func(is_haswell) sep \
-        func(is_broadwell) sep \
-        func(is_skylake) sep \
-        func(is_broxton) sep \
-        func(is_kabylake) sep \
-        func(is_preliminary) sep \
-        func(has_fbc) sep \
-        func(has_psr) sep \
-        func(has_runtime_pm) sep \
-        func(has_csr) sep \
-        func(has_resource_streamer) sep \
-        func(has_rc6) sep \
-        func(has_rc6p) sep \
-        func(has_dp_mst) sep \
-        func(has_gmbus_irq) sep \
-        func(has_hw_contexts) sep \
-        func(has_logical_ring_contexts) sep \
-        func(has_l3_dpf) sep \
-        func(has_gmch_display) sep \
-        func(has_guc) sep \
-        func(has_pipe_cxsr) sep \
-        func(has_hotplug) sep \
-        func(cursor_needs_physical) sep \
-        func(has_overlay) sep \
-        func(overlay_needs_physical) sep \
-        func(supports_tv) sep \
-        func(has_llc) sep \
-        func(has_snoop) sep \
-        func(has_ddi) sep \
-        func(has_fpga_dbg) sep \
-        func(has_pooled_eu)
-
-#define DEFINE_FLAG(name) u8 name:1
-#define SEP_SEMICOLON ;
+#define DEV_INFO_FOR_EACH_FLAG(func) \
+        /* Keep is_* in chronological order */ \
+        func(is_mobile); \
+        func(is_i85x); \
+        func(is_i915g); \
+        func(is_i945gm); \
+        func(is_g33); \
+        func(is_g4x); \
+        func(is_pineview); \
+        func(is_broadwater); \
+        func(is_crestline); \
+        func(is_ivybridge); \
+        func(is_valleyview); \
+        func(is_cherryview); \
+        func(is_haswell); \
+        func(is_broadwell); \
+        func(is_skylake); \
+        func(is_broxton); \
+        func(is_kabylake); \
+        func(is_preliminary); \
+        /* Keep has_* in alphabetical order */ \
+        func(has_csr); \
+        func(has_ddi); \
+        func(has_dp_mst); \
+        func(has_fbc); \
+        func(has_fpga_dbg); \
+        func(has_gmbus_irq); \
+        func(has_gmch_display); \
+        func(has_guc); \
+        func(has_hotplug); \
+        func(has_hw_contexts); \
+        func(has_l3_dpf); \
+        func(has_llc); \
+        func(has_logical_ring_contexts); \
+        func(has_overlay); \
+        func(has_pipe_cxsr); \
+        func(has_pooled_eu); \
+        func(has_psr); \
+        func(has_rc6); \
+        func(has_rc6p); \
+        func(has_resource_streamer); \
+        func(has_runtime_pm); \
+        func(has_snoop); \
+        func(cursor_needs_physical); \
+        func(hws_needs_physical); \
+        func(overlay_needs_physical); \
+        func(supports_tv)
 
 struct sseu_dev_info {
         u8 slice_mask;
@@ -709,7 +722,9 @@ struct intel_device_info {
         u16 gen_mask;
         u8 ring_mask; /* Rings supported by the HW */
         u8 num_rings;
-        DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON);
+#define DEFINE_FLAG(name) u8 name:1
+        DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
+#undef DEFINE_FLAG
         u16 ddb_size; /* in blocks */
         /* Register offsets for the various display pipes and transcoders */
         int pipe_offsets[I915_MAX_TRANSCODERS];
@@ -726,15 +741,14 @@ struct intel_device_info {
         } color;
 };
 
-#undef DEFINE_FLAG
-#undef SEP_SEMICOLON
-
 struct intel_display_error_state;
 
 struct drm_i915_error_state {
         struct kref ref;
         struct timeval time;
 
+        struct drm_i915_private *i915;
+
         char error_msg[128];
         bool simulated;
         int iommu;
@@ -759,7 +773,7 @@ struct drm_i915_error_state {
         u32 gam_ecochk;
         u32 gab_ctl;
         u32 gfx_mode;
-        u32 extra_instdone[I915_NUM_INSTDONE_REG];
+
         u64 fence[I915_MAX_NUM_FENCES];
         struct intel_overlay_error_state *overlay;
         struct intel_display_error_state *display;
@@ -775,6 +789,9 @@ struct drm_i915_error_state {
                 struct i915_address_space *vm;
                 int num_requests;
 
+                /* position of active request inside the ring */
+                u32 rq_head, rq_post, rq_tail;
+
                 /* our own tracking of ring head and tail */
                 u32 cpu_ring_head;
                 u32 cpu_ring_tail;
@@ -791,7 +808,6 @@ struct drm_i915_error_state {
                 u32 hws;
                 u32 ipeir;
                 u32 ipehr;
-                u32 instdone;
                 u32 bbstate;
                 u32 instpm;
                 u32 instps;
@@ -802,11 +818,13 @@ struct drm_i915_error_state {
                 u64 faddr;
                 u32 rc_psmi; /* sleep state */
                 u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
+                struct intel_instdone instdone;
 
                 struct drm_i915_error_object {
-                        int page_count;
                         u64 gtt_offset;
                         u64 gtt_size;
+                        int page_count;
+                        int unused;
                         u32 *pages[0];
                 } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
 
@@ -815,10 +833,11 @@ struct drm_i915_error_state {
                 struct drm_i915_error_request {
                         long jiffies;
                         pid_t pid;
+                        u32 context;
                         u32 seqno;
                         u32 head;
                         u32 tail;
-                } *requests;
+                } *requests, execlist[2];
 
                 struct drm_i915_error_waiter {
                         char comm[TASK_COMM_LEN];
@@ -972,6 +991,9 @@ struct intel_fbc {
         bool enabled;
         bool active;
 
+        bool underrun_detected;
+        struct work_struct underrun_work;
+
         struct intel_fbc_state_cache {
                 struct {
                         unsigned int mode_flags;
@@ -1368,7 +1390,7 @@ struct i915_gem_mm {
 
         /* accounting, useful for userland debugging */
         spinlock_t object_stat_lock;
-        size_t object_memory;
+        u64 object_memory;
         u32 object_count;
 };
 
@@ -1620,7 +1642,6 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1,
 }
 
 struct skl_ddb_allocation {
-        struct skl_ddb_entry pipe[I915_MAX_PIPES];
         struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */
         struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES];
 };
@@ -1628,15 +1649,12 @@ struct skl_ddb_allocation {
 struct skl_wm_values {
         unsigned dirty_pipes;
         struct skl_ddb_allocation ddb;
-        uint32_t wm_linetime[I915_MAX_PIPES];
-        uint32_t plane[I915_MAX_PIPES][I915_MAX_PLANES][8];
-        uint32_t plane_trans[I915_MAX_PIPES][I915_MAX_PLANES];
 };
 
 struct skl_wm_level {
-        bool plane_en[I915_MAX_PLANES];
-        uint16_t plane_res_b[I915_MAX_PLANES];
-        uint8_t plane_res_l[I915_MAX_PLANES];
+        bool plane_en;
+        uint16_t plane_res_b;
+        uint8_t plane_res_l;
 };
 
 /*
@@ -1759,7 +1777,7 @@ struct drm_i915_private {
 
         struct i915_virtual_gpu vgpu;
 
-        struct intel_gvt gvt;
+        struct intel_gvt *gvt;
 
         struct intel_guc guc;
 
@@ -1787,7 +1805,7 @@ struct drm_i915_private {
 
         struct pci_dev *bridge_dev;
         struct i915_gem_context *kernel_context;
-        struct intel_engine_cs engine[I915_NUM_ENGINES];
+        struct intel_engine_cs *engine[I915_NUM_ENGINES];
         struct i915_vma *semaphore;
         u32 next_seqno;
 
@@ -1984,11 +2002,11 @@ struct drm_i915_private {
         struct vlv_s0ix_state vlv_s0ix_state;
 
         enum {
-                I915_SKL_SAGV_UNKNOWN = 0,
-                I915_SKL_SAGV_DISABLED,
-                I915_SKL_SAGV_ENABLED,
-                I915_SKL_SAGV_NOT_CONTROLLED
-        } skl_sagv_status;
+                I915_SAGV_UNKNOWN = 0,
+                I915_SAGV_DISABLED,
+                I915_SAGV_ENABLED,
+                I915_SAGV_NOT_CONTROLLED
+        } sagv_status;
 
         struct {
                 /*
@@ -2079,7 +2097,8 @@ struct drm_i915_private {
         /* perform PHY state sanity checks? */
         bool chv_phy_assert[2];
 
-        struct intel_encoder *dig_port_map[I915_MAX_PORTS];
+        /* Used to save the pipe-to-encoder mapping for audio */
+        struct intel_encoder *av_enc_map[I915_MAX_PIPES];
 
         /*
          * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
@@ -2103,19 +2122,11 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 }
 
 /* Simple iterator over all initialised engines */
-#define for_each_engine(engine__, dev_priv__) \
-        for ((engine__) = &(dev_priv__)->engine[0]; \
-             (engine__) < &(dev_priv__)->engine[I915_NUM_ENGINES]; \
-             (engine__)++) \
-                for_each_if (intel_engine_initialized(engine__))
-
-/* Iterator with engine_id */
-#define for_each_engine_id(engine__, dev_priv__, id__) \
-        for ((engine__) = &(dev_priv__)->engine[0], (id__) = 0; \
-             (engine__) < &(dev_priv__)->engine[I915_NUM_ENGINES]; \
-             (engine__)++) \
-                for_each_if (((id__) = (engine__)->id, \
-                              intel_engine_initialized(engine__)))
+#define for_each_engine(engine__, dev_priv__, id__) \
+        for ((id__) = 0; \
+             (id__) < I915_NUM_ENGINES; \
+             (id__)++) \
+                for_each_if ((engine__) = (dev_priv__)->engine[(id__)])
 
 #define __mask_next_bit(mask) ({                                        \
         int __idx = ffs(mask) - 1;                                      \
@@ -2126,7 +2137,7 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 /* Iterator over subset of engines selected by mask */
 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
         for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask;        \
-             tmp__ ? (engine__ = &(dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
+             tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
 
 enum hdmi_force_audio {
         HDMI_AUDIO_OFF_DVI = -2,        /* no aux data for HDMI-DVI converter */
@@ -2276,21 +2287,19 @@ struct drm_i915_gem_object {
         /** Record of address bit 17 of each page at last unbind. */
         unsigned long *bit_17;
 
-        union {
-                /** for phy allocated objects */
-                struct drm_dma_handle *phys_handle;
-
-                struct i915_gem_userptr {
-                        uintptr_t ptr;
-                        unsigned read_only :1;
-                        unsigned workers :4;
+        struct i915_gem_userptr {
+                uintptr_t ptr;
+                unsigned read_only :1;
+                unsigned workers :4;
 #define I915_GEM_USERPTR_MAX_WORKERS 15
 
-                        struct i915_mm_struct *mm;
-                        struct i915_mmu_object *mmu_object;
-                        struct work_struct *work;
-                } userptr;
-        };
+                struct i915_mm_struct *mm;
+                struct i915_mmu_object *mmu_object;
+                struct work_struct *work;
+        } userptr;
+
+        /** for phys allocated objects */
+        struct drm_dma_handle *phys_handle;
 };
 
 static inline struct drm_i915_gem_object *
@@ -2588,8 +2597,9 @@ struct drm_i915_cmd_table {
         __p; \
 })
 #define INTEL_INFO(p)   (&__I915__(p)->info)
-#define INTEL_GEN(p)    (INTEL_INFO(p)->gen)
-#define INTEL_DEVID(p)  (INTEL_INFO(p)->device_id)
+
+#define INTEL_GEN(dev_priv)     ((dev_priv)->info.gen)
+#define INTEL_DEVID(dev_priv)   ((dev_priv)->info.device_id)
 
 #define REVID_FOREVER           0xff
 #define INTEL_REVID(p)  (__I915__(p)->drm.pdev->revision)
@@ -2600,7 +2610,7 @@ struct drm_i915_cmd_table {
  *
  * Use GEN_FOREVER for unbound start and or end.
  */
-#define IS_GEN(p, s, e) ({ \
+#define IS_GEN(dev_priv, s, e) ({ \
         unsigned int __s = (s), __e = (e); \
         BUILD_BUG_ON(!__builtin_constant_p(s)); \
         BUILD_BUG_ON(!__builtin_constant_p(e)); \
@@ -2610,7 +2620,7 @@ struct drm_i915_cmd_table {
                 __e = BITS_PER_LONG - 1; \
         else \
                 __e = (e) - 1; \
-        !!(INTEL_INFO(p)->gen_mask & GENMASK((__e), (__s))); \
+        !!((dev_priv)->info.gen_mask & GENMASK((__e), (__s))); \
 })
 
 /*
@@ -2621,73 +2631,73 @@ struct drm_i915_cmd_table {
 #define IS_REVID(p, since, until) \
         (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until))
 
-#define IS_I830(dev)            (INTEL_DEVID(dev) == 0x3577)
-#define IS_845G(dev)            (INTEL_DEVID(dev) == 0x2562)
+#define IS_I830(dev_priv)       (INTEL_DEVID(dev_priv) == 0x3577)
+#define IS_845G(dev_priv)       (INTEL_DEVID(dev_priv) == 0x2562)
 #define IS_I85X(dev)            (INTEL_INFO(dev)->is_i85x)
-#define IS_I865G(dev)           (INTEL_DEVID(dev) == 0x2572)
+#define IS_I865G(dev_priv)      (INTEL_DEVID(dev_priv) == 0x2572)
 #define IS_I915G(dev)           (INTEL_INFO(dev)->is_i915g)
-#define IS_I915GM(dev)          (INTEL_DEVID(dev) == 0x2592)
-#define IS_I945G(dev)           (INTEL_DEVID(dev) == 0x2772)
+#define IS_I915GM(dev_priv)     (INTEL_DEVID(dev_priv) == 0x2592)
+#define IS_I945G(dev_priv)      (INTEL_DEVID(dev_priv) == 0x2772)
 #define IS_I945GM(dev)          (INTEL_INFO(dev)->is_i945gm)
 #define IS_BROADWATER(dev)      (INTEL_INFO(dev)->is_broadwater)
 #define IS_CRESTLINE(dev)       (INTEL_INFO(dev)->is_crestline)
-#define IS_GM45(dev)            (INTEL_DEVID(dev) == 0x2A42)
-#define IS_G4X(dev)             (INTEL_INFO(dev)->is_g4x)
-#define IS_PINEVIEW_G(dev)      (INTEL_DEVID(dev) == 0xa001)
-#define IS_PINEVIEW_M(dev)      (INTEL_DEVID(dev) == 0xa011)
+#define IS_GM45(dev_priv)       (INTEL_DEVID(dev_priv) == 0x2A42)
+#define IS_G4X(dev_priv)        ((dev_priv)->info.is_g4x)
+#define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001)
+#define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011)
 #define IS_PINEVIEW(dev)        (INTEL_INFO(dev)->is_pineview)
 #define IS_G33(dev)             (INTEL_INFO(dev)->is_g33)
-#define IS_IRONLAKE_M(dev)      (INTEL_DEVID(dev) == 0x0046)
-#define IS_IVYBRIDGE(dev)       (INTEL_INFO(dev)->is_ivybridge)
-#define IS_IVB_GT1(dev)         (INTEL_DEVID(dev) == 0x0156 || \
-                                 INTEL_DEVID(dev) == 0x0152 || \
-                                 INTEL_DEVID(dev) == 0x015a)
-#define IS_VALLEYVIEW(dev)      (INTEL_INFO(dev)->is_valleyview)
-#define IS_CHERRYVIEW(dev)      (INTEL_INFO(dev)->is_cherryview)
-#define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell)
-#define IS_BROADWELL(dev)       (INTEL_INFO(dev)->is_broadwell)
-#define IS_SKYLAKE(dev) (INTEL_INFO(dev)->is_skylake)
-#define IS_BROXTON(dev)         (INTEL_INFO(dev)->is_broxton)
-#define IS_KABYLAKE(dev)        (INTEL_INFO(dev)->is_kabylake)
+#define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046)
+#define IS_IVYBRIDGE(dev_priv)  ((dev_priv)->info.is_ivybridge)
+#define IS_IVB_GT1(dev_priv)    (INTEL_DEVID(dev_priv) == 0x0156 || \
+                                 INTEL_DEVID(dev_priv) == 0x0152 || \
+                                 INTEL_DEVID(dev_priv) == 0x015a)
+#define IS_VALLEYVIEW(dev_priv) ((dev_priv)->info.is_valleyview)
+#define IS_CHERRYVIEW(dev_priv) ((dev_priv)->info.is_cherryview)
+#define IS_HASWELL(dev_priv)    ((dev_priv)->info.is_haswell)
+#define IS_BROADWELL(dev_priv)  ((dev_priv)->info.is_broadwell)
+#define IS_SKYLAKE(dev_priv)    ((dev_priv)->info.is_skylake)
+#define IS_BROXTON(dev_priv)    ((dev_priv)->info.is_broxton)
+#define IS_KABYLAKE(dev_priv)   ((dev_priv)->info.is_kabylake)
 #define IS_MOBILE(dev)          (INTEL_INFO(dev)->is_mobile)
-#define IS_HSW_EARLY_SDV(dev)   (IS_HASWELL(dev) && \
-                                 (INTEL_DEVID(dev) & 0xFF00) == 0x0C00)
-#define IS_BDW_ULT(dev)         (IS_BROADWELL(dev) && \
-                                 ((INTEL_DEVID(dev) & 0xf) == 0x6 ||    \
-                                 (INTEL_DEVID(dev) & 0xf) == 0xb ||     \
-                                 (INTEL_DEVID(dev) & 0xf) == 0xe))
+#define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
+                                    (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
+#define IS_BDW_ULT(dev_priv)    (IS_BROADWELL(dev_priv) && \
+                                 ((INTEL_DEVID(dev_priv) & 0xf) == 0x6 ||       \
+                                 (INTEL_DEVID(dev_priv) & 0xf) == 0xb ||        \
+                                 (INTEL_DEVID(dev_priv) & 0xf) == 0xe))
 /* ULX machines are also considered ULT. */
-#define IS_BDW_ULX(dev)         (IS_BROADWELL(dev) && \
-                                 (INTEL_DEVID(dev) & 0xf) == 0xe)
-#define IS_BDW_GT3(dev)         (IS_BROADWELL(dev) && \
-                                 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
-#define IS_HSW_ULT(dev)         (IS_HASWELL(dev) && \
-                                 (INTEL_DEVID(dev) & 0xFF00) == 0x0A00)
-#define IS_HSW_GT3(dev)         (IS_HASWELL(dev) && \
-                                 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
+#define IS_BDW_ULX(dev_priv)    (IS_BROADWELL(dev_priv) && \
+                                 (INTEL_DEVID(dev_priv) & 0xf) == 0xe)
+#define IS_BDW_GT3(dev_priv)    (IS_BROADWELL(dev_priv) && \
+                                 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
+#define IS_HSW_ULT(dev_priv)    (IS_HASWELL(dev_priv) && \
+                                 (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00)
+#define IS_HSW_GT3(dev_priv)    (IS_HASWELL(dev_priv) && \
+                                 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
 /* ULX machines are also considered ULT. */
-#define IS_HSW_ULX(dev)         (INTEL_DEVID(dev) == 0x0A0E || \
-                                 INTEL_DEVID(dev) == 0x0A1E)
-#define IS_SKL_ULT(dev)         (INTEL_DEVID(dev) == 0x1906 || \
-                                 INTEL_DEVID(dev) == 0x1913 || \
-                                 INTEL_DEVID(dev) == 0x1916 || \
-                                 INTEL_DEVID(dev) == 0x1921 || \
-                                 INTEL_DEVID(dev) == 0x1926)
-#define IS_SKL_ULX(dev)         (INTEL_DEVID(dev) == 0x190E || \
-                                 INTEL_DEVID(dev) == 0x1915 || \
-                                 INTEL_DEVID(dev) == 0x191E)
-#define IS_KBL_ULT(dev)         (INTEL_DEVID(dev) == 0x5906 || \
-                                 INTEL_DEVID(dev) == 0x5913 || \
-                                 INTEL_DEVID(dev) == 0x5916 || \
-                                 INTEL_DEVID(dev) == 0x5921 || \
-                                 INTEL_DEVID(dev) == 0x5926)
-#define IS_KBL_ULX(dev)         (INTEL_DEVID(dev) == 0x590E || \
-                                 INTEL_DEVID(dev) == 0x5915 || \
-                                 INTEL_DEVID(dev) == 0x591E)
-#define IS_SKL_GT3(dev)         (IS_SKYLAKE(dev) && \
-                                 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
-#define IS_SKL_GT4(dev)         (IS_SKYLAKE(dev) && \
-                                 (INTEL_DEVID(dev) & 0x00F0) == 0x0030)
+#define IS_HSW_ULX(dev_priv)    (INTEL_DEVID(dev_priv) == 0x0A0E || \
+                                 INTEL_DEVID(dev_priv) == 0x0A1E)
+#define IS_SKL_ULT(dev_priv)    (INTEL_DEVID(dev_priv) == 0x1906 || \
+                                 INTEL_DEVID(dev_priv) == 0x1913 || \
+                                 INTEL_DEVID(dev_priv) == 0x1916 || \
+                                 INTEL_DEVID(dev_priv) == 0x1921 || \
+                                 INTEL_DEVID(dev_priv) == 0x1926)
+#define IS_SKL_ULX(dev_priv)    (INTEL_DEVID(dev_priv) == 0x190E || \
+                                 INTEL_DEVID(dev_priv) == 0x1915 || \
+                                 INTEL_DEVID(dev_priv) == 0x191E)
+#define IS_KBL_ULT(dev_priv)    (INTEL_DEVID(dev_priv) == 0x5906 || \
+                                 INTEL_DEVID(dev_priv) == 0x5913 || \
+                                 INTEL_DEVID(dev_priv) == 0x5916 || \
+                                 INTEL_DEVID(dev_priv) == 0x5921 || \
+                                 INTEL_DEVID(dev_priv) == 0x5926)
+#define IS_KBL_ULX(dev_priv)    (INTEL_DEVID(dev_priv) == 0x590E || \
+                                 INTEL_DEVID(dev_priv) == 0x5915 || \
+                                 INTEL_DEVID(dev_priv) == 0x591E)
+#define IS_SKL_GT3(dev_priv)    (IS_SKYLAKE(dev_priv) && \
+                                 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
+#define IS_SKL_GT4(dev_priv)    (IS_SKYLAKE(dev_priv) && \
+                                 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030)
 
 #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary)
 
@@ -2707,7 +2717,8 @@ struct drm_i915_cmd_table {
 #define BXT_REVID_B0            0x3
 #define BXT_REVID_C0            0x9
 
-#define IS_BXT_REVID(p, since, until) (IS_BROXTON(p) && IS_REVID(p, since, until))
+#define IS_BXT_REVID(dev_priv, since, until) \
+        (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until))
 
 #define KBL_REVID_A0            0x0
 #define KBL_REVID_B0            0x1
@@ -2715,8 +2726,8 @@ struct drm_i915_cmd_table {
 #define KBL_REVID_D0            0x3
 #define KBL_REVID_E0            0x4
 
-#define IS_KBL_REVID(p, since, until) \
-        (IS_KABYLAKE(p) && IS_REVID(p, since, until))
+#define IS_KBL_REVID(dev_priv, since, until) \
+        (IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until))
 
 /*
  * The genX designation typically refers to the render engine, so render
@@ -2724,14 +2735,14 @@ struct drm_i915_cmd_table {
  * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular
  * chips, etc.).
  */
-#define IS_GEN2(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(1)))
-#define IS_GEN3(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(2)))
-#define IS_GEN4(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(3)))
-#define IS_GEN5(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(4)))
-#define IS_GEN6(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(5)))
-#define IS_GEN7(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(6)))
-#define IS_GEN8(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(7)))
-#define IS_GEN9(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(8)))
+#define IS_GEN2(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(1)))
+#define IS_GEN3(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(2)))
+#define IS_GEN4(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(3)))
+#define IS_GEN5(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(4)))
+#define IS_GEN6(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(5)))
+#define IS_GEN7(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(6)))
+#define IS_GEN8(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(7)))
+#define IS_GEN9(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(8)))
 
 #define ENGINE_MASK(id) BIT(id)
 #define RENDER_RING     ENGINE_MASK(RCS)
@@ -2752,8 +2763,8 @@ struct drm_i915_cmd_table {
 #define HAS_LLC(dev)            (INTEL_INFO(dev)->has_llc)
 #define HAS_SNOOP(dev)          (INTEL_INFO(dev)->has_snoop)
 #define HAS_EDRAM(dev)          (!!(__I915__(dev)->edram_cap & EDRAM_ENABLED))
-#define HAS_WT(dev)             ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && \
-                                 HAS_EDRAM(dev))
+#define HAS_WT(dev_priv)        ((IS_HASWELL(dev_priv) || \
+                                 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv))
 #define HWS_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->hws_needs_physical)
 
 #define HAS_HW_CONTEXTS(dev)    (INTEL_INFO(dev)->has_hw_contexts)
@@ -2766,7 +2777,7 @@ struct drm_i915_cmd_table {
 #define OVERLAY_NEEDS_PHYSICAL(dev)     (INTEL_INFO(dev)->overlay_needs_physical)
 
 /* Early gen2 have a totally busted CS tlb and require pinned batches. */
-#define HAS_BROKEN_CS_TLB(dev)          (IS_I830(dev) || IS_845G(dev))
+#define HAS_BROKEN_CS_TLB(dev_priv)     (IS_I830(dev_priv) || IS_845G(dev_priv))
 
 /* WaRsDisableCoarsePowerGating:skl,bxt */
 #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
@@ -2786,8 +2797,9 @@ struct drm_i915_cmd_table {
 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
  * rows, which changed the alignment requirements and fence programming.
  */
-#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
-                                                      IS_I915GM(dev)))
+#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \
+                                         !(IS_I915G(dev_priv) || \
+                                         IS_I915GM(dev_priv)))
 #define SUPPORTS_TV(dev)                (INTEL_INFO(dev)->supports_tv)
 #define I915_HAS_HOTPLUG(dev)            (INTEL_INFO(dev)->has_hotplug)
 
@@ -2795,19 +2807,19 @@ struct drm_i915_cmd_table {
 #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
 #define HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
 
-#define HAS_IPS(dev)            (IS_HSW_ULT(dev) || IS_BROADWELL(dev))
+#define HAS_IPS(dev_priv)       (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv))
 
 #define HAS_DP_MST(dev) (INTEL_INFO(dev)->has_dp_mst)
 
-#define HAS_DDI(dev)            (INTEL_INFO(dev)->has_ddi)
+#define HAS_DDI(dev_priv)       ((dev_priv)->info.has_ddi)
 #define HAS_FPGA_DBG_UNCLAIMED(dev)     (INTEL_INFO(dev)->has_fpga_dbg)
 #define HAS_PSR(dev)            (INTEL_INFO(dev)->has_psr)
-#define HAS_RUNTIME_PM(dev)     (INTEL_INFO(dev)->has_runtime_pm)
 #define HAS_RC6(dev)            (INTEL_INFO(dev)->has_rc6)
 #define HAS_RC6p(dev)           (INTEL_INFO(dev)->has_rc6p)
 
 #define HAS_CSR(dev)    (INTEL_INFO(dev)->has_csr)
 
+#define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm)
 /*
  * For now, anything with a GuC requires uCode loading, and then supports
  * command submission once loaded. But these are logically independent
@@ -2834,22 +2846,27 @@ struct drm_i915_cmd_table {
 #define INTEL_PCH_P3X_DEVICE_ID_TYPE            0x7000
 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE           0x2900 /* qemu q35 has 2918 */
 
-#define INTEL_PCH_TYPE(dev) (__I915__(dev)->pch_type)
-#define HAS_PCH_KBP(dev) (INTEL_PCH_TYPE(dev) == PCH_KBP)
-#define HAS_PCH_SPT(dev) (INTEL_PCH_TYPE(dev) == PCH_SPT)
-#define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT)
-#define HAS_PCH_LPT_LP(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
-#define HAS_PCH_LPT_H(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
-#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
-#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
-#define HAS_PCH_NOP(dev) (INTEL_PCH_TYPE(dev) == PCH_NOP)
-#define HAS_PCH_SPLIT(dev) (INTEL_PCH_TYPE(dev) != PCH_NONE)
+#define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type)
+#define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP)
+#define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT)
+#define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT)
+#define HAS_PCH_LPT_LP(dev_priv) \
+        ((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
+#define HAS_PCH_LPT_H(dev_priv) \
+        ((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
+#define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT)
+#define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX)
+#define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP)
+#define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE)
+
+#define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display)
 
-#define HAS_GMCH_DISPLAY(dev) (INTEL_INFO(dev)->has_gmch_display)
+#define HAS_LSPCON(dev_priv) (IS_GEN9(dev_priv))
 
 /* DPF == dynamic parity feature */
-#define HAS_L3_DPF(dev) (INTEL_INFO(dev)->has_l3_dpf)
-#define NUM_L3_SLICES(dev) (IS_HSW_GT3(dev) ? 2 : HAS_L3_DPF(dev))
+#define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf)
+#define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \
+                                 2 : HAS_L3_DPF(dev_priv))
 
 #define GT_FREQUENCY_MULTIPLIER 50
 #define GEN9_FREQ_SCALER 3
@@ -2885,6 +2902,11 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
                               unsigned long arg);
 #endif
+extern const struct dev_pm_ops i915_pm_ops;
+
+extern int i915_driver_load(struct pci_dev *pdev,
+                            const struct pci_device_id *ent);
+extern void i915_driver_unload(struct drm_device *dev);
 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask);
 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
 extern void i915_reset(struct drm_i915_private *dev_priv);
@@ -2971,7 +2993,7 @@ int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
 
 static inline bool intel_gvt_active(struct drm_i915_private *dev_priv)
 {
-        return dev_priv->gvt.initialized;
+        return dev_priv->gvt;
 }
 
 static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv)
@@ -3076,6 +3098,7 @@ int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
 void i915_gem_load_init(struct drm_device *dev);
 void i915_gem_load_cleanup(struct drm_device *dev);
 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
+int i915_gem_freeze(struct drm_i915_private *dev_priv);
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
 
 void *i915_gem_object_alloc(struct drm_device *dev);
@@ -3083,7 +3106,7 @@ void i915_gem_object_free(struct drm_i915_gem_object *obj);
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
                          const struct drm_i915_gem_object_ops *ops);
 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
-                                                  size_t size);
+                                                   u64 size);
 struct drm_i915_gem_object *i915_gem_object_create_from_data(
                 struct drm_device *dev, const void *data, size_t size);
 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file);
@@ -3157,14 +3180,15 @@ i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
 
 static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
-        BUG_ON(obj->pages == NULL);
+        GEM_BUG_ON(obj->pages == NULL);
         obj->pages_pin_count++;
 }
 
 static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 {
-        BUG_ON(obj->pages_pin_count == 0);
+        GEM_BUG_ON(obj->pages_pin_count == 0);
         obj->pages_pin_count--;
+        GEM_BUG_ON(obj->pages_pin_count < obj->bind_count);
 }
 
 enum i915_map_type {
@@ -3522,6 +3546,8 @@ static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}
 #endif
 
 /* i915_gpu_error.c */
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 __printf(2, 3)
 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
 int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
@@ -3542,7 +3568,20 @@ void i915_error_state_get(struct drm_device *dev,
 void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
 void i915_destroy_error_state(struct drm_device *dev);
 
-void i915_get_extra_instdone(struct drm_i915_private *dev_priv, uint32_t *instdone);
+#else
+
+static inline void i915_capture_error_state(struct drm_i915_private *dev_priv,
+                                            u32 engine_mask,
+                                            const char *error_msg)
+{
+}
+
+static inline void i915_destroy_error_state(struct drm_device *dev)
+{
+}
+
+#endif
+
 const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
 
 /* i915_cmd_parser.c */
@@ -3592,6 +3631,9 @@ bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum por
 bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, enum port *port);
 bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv,
                                      enum port port);
+bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv,
+                                enum port port);
+
 
 /* intel_opregion.c */
 #ifdef CONFIG_ACPI
@@ -3808,11 +3850,11 @@ __raw_write(64, q)
 #define INTEL_BROADCAST_RGB_FULL 1
 #define INTEL_BROADCAST_RGB_LIMITED 2
 
-static inline i915_reg_t i915_vgacntrl_reg(struct drm_device *dev)
+static inline i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv)
 {
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 return VLV_VGACNTRL;
-        else if (INTEL_INFO(dev)->gen >= 5)
+        else if (INTEL_GEN(dev_priv) >= 5)
                 return CPU_VGACNTRL;
         else
                 return VGACNTRL;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 2c8106758922..8ed8e24025ac 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -82,7 +82,7 @@ remove_mappable_node(struct drm_mm_node *node)
 
 /* some bookkeeping */
 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
-                                  size_t size)
+                                  u64 size)
 {
         spin_lock(&dev_priv->mm.object_stat_lock);
         dev_priv->mm.object_count++;
@@ -91,7 +91,7 @@ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 }
 
 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
-                                     size_t size)
+                                     u64 size)
 {
         spin_lock(&dev_priv->mm.object_stat_lock);
         dev_priv->mm.object_count--;
@@ -860,7 +860,7 @@ i915_gem_gtt_pread(struct drm_device *dev,
 
         mutex_unlock(&dev->struct_mutex);
         if (likely(!i915.prefault_disable)) {
-                ret = fault_in_multipages_writeable(user_data, remain);
+                ret = fault_in_pages_writeable(user_data, remain);
                 if (ret) {
                         mutex_lock(&dev->struct_mutex);
                         goto out_unpin;
@@ -919,8 +919,7 @@ out_unpin:
         if (node.allocated) {
                 wmb();
                 ggtt->base.clear_range(&ggtt->base,
-                                       node.start, node.size,
-                                       true);
+                                       node.start, node.size);
                 i915_gem_object_unpin_pages(obj);
                 remove_mappable_node(&node);
         } else {
@@ -983,7 +982,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
                 mutex_unlock(&dev->struct_mutex);
 
                 if (likely(!i915.prefault_disable) && !prefaulted) {
-                        ret = fault_in_multipages_writeable(user_data, remain);
+                        ret = fault_in_pages_writeable(user_data, remain);
                         /* Userspace is tricking us, but we've already clobbered
                          * its pages with the prefault and promised to write the
                          * data up to the first fault. Hence ignore any errors
@@ -1228,8 +1227,7 @@ out_unpin:
         if (node.allocated) {
                 wmb();
                 ggtt->base.clear_range(&ggtt->base,
-                                       node.start, node.size,
-                                       true);
+                                       node.start, node.size);
                 i915_gem_object_unpin_pages(obj);
                 remove_mappable_node(&node);
         } else {
@@ -1431,7 +1429,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                 return -EFAULT;
 
         if (likely(!i915.prefault_disable)) {
-                ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
+                ret = fault_in_pages_readable(u64_to_user_ptr(args->data_ptr),
                                                    args->size);
                 if (ret)
                         return -EFAULT;
@@ -1813,8 +1811,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
                 view.params.partial.offset = rounddown(page_offset, chunk_size);
                 view.params.partial.size =
                         min_t(unsigned int, chunk_size,
-                              (area->vm_end - area->vm_start) / PAGE_SIZE -
-                              view.params.partial.offset);
+                              vma_pages(area) - view.params.partial.offset);
 
                 /* If the partial covers the entire object, just create a
                  * normal VMA.
@@ -2208,6 +2205,15 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
         return 0;
 }
 
+static unsigned int swiotlb_max_size(void)
+{
+#if IS_ENABLED(CONFIG_SWIOTLB)
+        return rounddown(swiotlb_nr_tbl() << IO_TLB_SHIFT, PAGE_SIZE);
+#else
+        return 0;
+#endif
+}
+
 static int
 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 {
@@ -2219,6 +2225,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
         struct sgt_iter sgt_iter;
         struct page *page;
         unsigned long last_pfn = 0;     /* suppress gcc warning */
+        unsigned int max_segment;
         int ret;
         gfp_t gfp;
 
@@ -2229,6 +2236,10 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
 
+        max_segment = swiotlb_max_size();
+        if (!max_segment)
+                max_segment = rounddown(UINT_MAX, PAGE_SIZE);
+
         st = kmalloc(sizeof(*st), GFP_KERNEL);
         if (st == NULL)
                 return -ENOMEM;
@@ -2264,22 +2275,15 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
                          * our own buffer, now let the real VM do its job and
                          * go down in flames if truly OOM.
                          */
-                        i915_gem_shrink_all(dev_priv);
                         page = shmem_read_mapping_page(mapping, i);
                         if (IS_ERR(page)) {
                                 ret = PTR_ERR(page);
                                 goto err_pages;
                         }
                 }
-#ifdef CONFIG_SWIOTLB
-                if (swiotlb_nr_tbl()) {
-                        st->nents++;
-                        sg_set_page(sg, page, PAGE_SIZE, 0);
-                        sg = sg_next(sg);
-                        continue;
-                }
-#endif
-                if (!i || page_to_pfn(page) != last_pfn + 1) {
+                if (!i ||
+                    sg->length >= max_segment ||
+                    page_to_pfn(page) != last_pfn + 1) {
                         if (i)
                                 sg = sg_next(sg);
                         st->nents++;
@@ -2292,9 +2296,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
                 /* Check that the i965g/gm workaround works. */
                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
         }
-#ifdef CONFIG_SWIOTLB
-        if (!swiotlb_nr_tbl())
-#endif
+        if (sg) /* loop terminated early; short sg table */
                 sg_mark_end(sg);
         obj->pages = st;
 
@@ -2581,8 +2583,6 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine)
         struct i915_gem_context *incomplete_ctx;
         bool ring_hung;
 
-        /* Ensure irq handler finishes, and not run again. */
-        tasklet_kill(&engine->irq_tasklet);
         if (engine->irq_seqno_barrier)
                 engine->irq_seqno_barrier(engine);
 
@@ -2591,6 +2591,9 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine)
                 return;
 
         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
+        if (engine->hangcheck.seqno != intel_engine_get_seqno(engine))
+                ring_hung = false;
+
         i915_set_reset_status(request->ctx, ring_hung);
         if (!ring_hung)
                 return;
@@ -2616,21 +2619,26 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine)
         list_for_each_entry_continue(request, &engine->request_list, link)
                 if (request->ctx == incomplete_ctx)
                         reset_request(request);
-
-        engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
 }
 
 void i915_gem_reset(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
         i915_gem_retire_requests(dev_priv);
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 i915_gem_reset_engine(engine);
-        mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
 
         i915_gem_restore_fences(&dev_priv->drm);
+
+        if (dev_priv->gt.awake) {
+                intel_sanitize_gt_powersave(dev_priv);
+                intel_enable_gt_powersave(dev_priv);
+                if (INTEL_GEN(dev_priv) >= 6)
+                        gen6_rps_busy(dev_priv);
+        }
 }
 
 static void nop_submit_request(struct drm_i915_gem_request *request)
@@ -2668,12 +2676,13 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
 void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
         lockdep_assert_held(&dev_priv->drm.struct_mutex);
         set_bit(I915_WEDGED, &dev_priv->gpu_error.flags);
 
         i915_gem_context_lost(dev_priv);
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 i915_gem_cleanup_engine(engine);
         mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
 
@@ -2712,6 +2721,7 @@ i915_gem_idle_work_handler(struct work_struct *work)
                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
         struct drm_device *dev = &dev_priv->drm;
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         bool rearm_hangcheck;
 
         if (!READ_ONCE(dev_priv->gt.awake))
@@ -2734,7 +2744,7 @@ i915_gem_idle_work_handler(struct work_struct *work)
         if (dev_priv->gt.active_engines)
                 goto out_unlock;
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 i915_gem_batch_pool_fini(&engine->batch_pool);
 
         GEM_BUG_ON(!dev_priv->gt.awake);
@@ -2927,9 +2937,10 @@ int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
                            unsigned int flags)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int ret;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 if (engine->last_context == NULL)
                         continue;
 
@@ -3091,6 +3102,9 @@ search_free:
 
                         goto err_unpin;
                 }
+
+                GEM_BUG_ON(vma->node.start < start);
+                GEM_BUG_ON(vma->node.start + vma->node.size > end);
         }
         GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
 
@@ -3169,7 +3183,7 @@ i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
          */
         wmb();
         if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv))
-                POSTING_READ(RING_ACTHD(dev_priv->engine[RCS].mmio_base));
+                POSTING_READ(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
 
         intel_fb_obj_flush(obj, false, write_origin(obj, I915_GEM_DOMAIN_GTT));
 
@@ -3464,7 +3478,7 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
                 level = I915_CACHE_LLC;
                 break;
         case I915_CACHING_DISPLAY:
-                level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
+                level = HAS_WT(dev_priv) ? I915_CACHE_WT : I915_CACHE_NONE;
                 break;
         default:
                 return -EINVAL;
@@ -3522,7 +3536,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
          */
         ret = i915_gem_object_set_cache_level(obj,
-                                              HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
+                                              HAS_WT(to_i915(obj->base.dev)) ?
+                                              I915_CACHE_WT : I915_CACHE_NONE);
         if (ret) {
                 vma = ERR_PTR(ret);
                 goto err_unpin_display;
@@ -3789,7 +3804,8 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
                          u64 alignment,
                          u64 flags)
 {
-        struct i915_address_space *vm = &to_i915(obj->base.dev)->ggtt.base;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+        struct i915_address_space *vm = &dev_priv->ggtt.base;
         struct i915_vma *vma;
         int ret;
 
@@ -3802,6 +3818,41 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
                     (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
                         return ERR_PTR(-ENOSPC);
 
+                if (flags & PIN_MAPPABLE) {
+                        u32 fence_size;
+
+                        fence_size = i915_gem_get_ggtt_size(dev_priv, vma->size,
+                                                            i915_gem_object_get_tiling(obj));
+                        /* If the required space is larger than the available
+                         * aperture, we will not able to find a slot for the
+                         * object and unbinding the object now will be in
+                         * vain. Worse, doing so may cause us to ping-pong
+                         * the object in and out of the Global GTT and
+                         * waste a lot of cycles under the mutex.
+                         */
+                        if (fence_size > dev_priv->ggtt.mappable_end)
+                                return ERR_PTR(-E2BIG);
+
+                        /* If NONBLOCK is set the caller is optimistically
+                         * trying to cache the full object within the mappable
+                         * aperture, and *must* have a fallback in place for
+                         * situations where we cannot bind the object. We
+                         * can be a little more lax here and use the fallback
+                         * more often to avoid costly migrations of ourselves
+                         * and other objects within the aperture.
+                         *
+                         * Half-the-aperture is used as a simple heuristic.
+                         * More interesting would to do search for a free
+                         * block prior to making the commitment to unbind.
+                         * That caters for the self-harm case, and with a
+                         * little more heuristics (e.g. NOFAULT, NOEVICT)
+                         * we could try to minimise harm to others.
+                         */
+                        if (flags & PIN_NONBLOCK &&
+                            fence_size > dev_priv->ggtt.mappable_end / 2)
+                                return ERR_PTR(-ENOSPC);
+                }
+
                 WARN(i915_vma_is_pinned(vma),
                      "bo is already pinned in ggtt with incorrect alignment:"
                      " offset=%08x, req.alignment=%llx,"
@@ -4080,14 +4131,29 @@ static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
         .put_pages = i915_gem_object_put_pages_gtt,
 };
 
-struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
-                                                  size_t size)
+/* Note we don't consider signbits :| */
+#define overflows_type(x, T) \
+        (sizeof(x) > sizeof(T) && (x) >> (sizeof(T) * BITS_PER_BYTE))
+
+struct drm_i915_gem_object *
+i915_gem_object_create(struct drm_device *dev, u64 size)
 {
         struct drm_i915_gem_object *obj;
         struct address_space *mapping;
         gfp_t mask;
         int ret;
 
+        /* There is a prevalence of the assumption that we fit the object's
+         * page count inside a 32bit _signed_ variable. Let's document this and
+         * catch if we ever need to fix it. In the meantime, if you do spot
+         * such a local variable, please consider fixing!
+         */
+        if (WARN_ON(size >> PAGE_SHIFT > INT_MAX))
+                return ERR_PTR(-E2BIG);
+
+        if (overflows_type(size, obj->base.size))
+                return ERR_PTR(-E2BIG);
+
         obj = i915_gem_object_alloc(dev);
         if (obj == NULL)
                 return ERR_PTR(-ENOMEM);
@@ -4264,6 +4330,30 @@ int i915_gem_suspend(struct drm_device *dev)
          */
         WARN_ON(dev_priv->gt.awake);
 
+        /*
+         * Neither the BIOS, ourselves or any other kernel
+         * expects the system to be in execlists mode on startup,
+         * so we need to reset the GPU back to legacy mode. And the only
+         * known way to disable logical contexts is through a GPU reset.
+         *
+         * So in order to leave the system in a known default configuration,
+         * always reset the GPU upon unload and suspend. Afterwards we then
+         * clean up the GEM state tracking, flushing off the requests and
+         * leaving the system in a known idle state.
+         *
+         * Note that is of the upmost importance that the GPU is idle and
+         * all stray writes are flushed *before* we dismantle the backing
+         * storage for the pinned objects.
+         *
+         * However, since we are uncertain that resetting the GPU on older
+         * machines is a good idea, we don't - just in case it leaves the
+         * machine in an unusable condition.
+         */
+        if (HAS_HW_CONTEXTS(dev)) {
+                int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
+                WARN_ON(reset && reset != -ENODEV);
+        }
+
         return 0;
 
 err:
@@ -4298,44 +4388,42 @@ void i915_gem_init_swizzling(struct drm_device *dev)
         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
                                  DISP_TILE_SURFACE_SWIZZLING);
 
-        if (IS_GEN5(dev))
+        if (IS_GEN5(dev_priv))
                 return;
 
         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev_priv))
                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
-        else if (IS_GEN7(dev))
+        else if (IS_GEN7(dev_priv))
                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
-        else if (IS_GEN8(dev))
+        else if (IS_GEN8(dev_priv))
                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
         else
                 BUG();
 }
 
-static void init_unused_ring(struct drm_device *dev, u32 base)
+static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
-
         I915_WRITE(RING_CTL(base), 0);
         I915_WRITE(RING_HEAD(base), 0);
         I915_WRITE(RING_TAIL(base), 0);
         I915_WRITE(RING_START(base), 0);
 }
 
-static void init_unused_rings(struct drm_device *dev)
-{
-        if (IS_I830(dev)) {
-                init_unused_ring(dev, PRB1_BASE);
-                init_unused_ring(dev, SRB0_BASE);
-                init_unused_ring(dev, SRB1_BASE);
-                init_unused_ring(dev, SRB2_BASE);
-                init_unused_ring(dev, SRB3_BASE);
-        } else if (IS_GEN2(dev)) {
-                init_unused_ring(dev, SRB0_BASE);
-                init_unused_ring(dev, SRB1_BASE);
-        } else if (IS_GEN3(dev)) {
-                init_unused_ring(dev, PRB1_BASE);
-                init_unused_ring(dev, PRB2_BASE);
+static void init_unused_rings(struct drm_i915_private *dev_priv)
+{
+        if (IS_I830(dev_priv)) {
+                init_unused_ring(dev_priv, PRB1_BASE);
+                init_unused_ring(dev_priv, SRB0_BASE);
+                init_unused_ring(dev_priv, SRB1_BASE);
+                init_unused_ring(dev_priv, SRB2_BASE);
+                init_unused_ring(dev_priv, SRB3_BASE);
+        } else if (IS_GEN2(dev_priv)) {
+                init_unused_ring(dev_priv, SRB0_BASE);
+                init_unused_ring(dev_priv, SRB1_BASE);
+        } else if (IS_GEN3(dev_priv)) {
+                init_unused_ring(dev_priv, PRB1_BASE);
+                init_unused_ring(dev_priv, PRB2_BASE);
         }
 }
 
@@ -4344,6 +4432,7 @@ i915_gem_init_hw(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int ret;
 
         /* Double layer security blanket, see i915_gem_init() */
@@ -4352,12 +4441,12 @@ i915_gem_init_hw(struct drm_device *dev)
         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
 
-        if (IS_HASWELL(dev))
-                I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
+        if (IS_HASWELL(dev_priv))
+                I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
 
-        if (HAS_PCH_NOP(dev)) {
-                if (IS_IVYBRIDGE(dev)) {
+        if (HAS_PCH_NOP(dev_priv)) {
+                if (IS_IVYBRIDGE(dev_priv)) {
                         u32 temp = I915_READ(GEN7_MSG_CTL);
                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
                         I915_WRITE(GEN7_MSG_CTL, temp);
@@ -4376,7 +4465,7 @@ i915_gem_init_hw(struct drm_device *dev)
          * will prevent c3 entry. Makes sure all unused rings
          * are totally idle.
          */
-        init_unused_rings(dev);
+        init_unused_rings(dev_priv);
 
         BUG_ON(!dev_priv->kernel_context);
 
@@ -4387,7 +4476,7 @@ i915_gem_init_hw(struct drm_device *dev)
         }
 
         /* Need to do basic initialisation of all rings first: */
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 ret = engine->init_hw(engine);
                 if (ret)
                         goto out;
@@ -4486,17 +4575,12 @@ i915_gem_cleanup_engines(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 dev_priv->gt.cleanup_engine(engine);
 }
 
-static void
-init_engine_lists(struct intel_engine_cs *engine)
-{
-        INIT_LIST_HEAD(&engine->request_list);
-}
-
 void
 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 {
@@ -4533,7 +4617,6 @@ void
 i915_gem_load_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
-        int i;
 
         dev_priv->objects =
                 kmem_cache_create("i915_gem_object",
@@ -4557,8 +4640,6 @@ i915_gem_load_init(struct drm_device *dev)
         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-        for (i = 0; i < I915_NUM_ENGINES; i++)
-                init_engine_lists(&dev_priv->engine[i]);
         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
                           i915_gem_retire_work_handler);
         INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
@@ -4589,6 +4670,19 @@ void i915_gem_load_cleanup(struct drm_device *dev)
         rcu_barrier();
 }
 
+int i915_gem_freeze(struct drm_i915_private *dev_priv)
+{
+        intel_runtime_pm_get(dev_priv);
+
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        i915_gem_shrink_all(dev_priv);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+
+        intel_runtime_pm_put(dev_priv);
+
+        return 0;
+}
+
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
 {
         struct drm_i915_gem_object *obj;
@@ -4612,7 +4706,8 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
          * the objects as well.
          */
 
-        i915_gem_shrink_all(dev_priv);
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND);
 
         for (p = phases; *p; p++) {
                 list_for_each_entry(obj, *p, global_list) {
@@ -4620,6 +4715,7 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
                         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
                 }
         }
+        mutex_unlock(&dev_priv->drm.struct_mutex);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index df10f4e95736..5dca32ac1c67 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -192,7 +192,7 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
          * This is only applicable for Ivy Bridge devices since
          * later platforms don't have L3 control bits in the PTE.
          */
-        if (IS_IVYBRIDGE(dev)) {
+        if (IS_IVYBRIDGE(to_i915(dev))) {
                 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
                 /* Failure shouldn't ever happen this early */
                 if (WARN_ON(ret)) {
@@ -474,10 +474,11 @@ int i915_gem_context_init(struct drm_device *dev)
 void i915_gem_context_lost(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
         lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 if (engine->last_context) {
                         i915_gem_context_unpin(engine->last_context, engine);
                         engine->last_context = NULL;
@@ -492,13 +493,13 @@ void i915_gem_context_lost(struct drm_i915_private *dev_priv)
                         if (!i915_gem_context_is_default(ctx))
                                 continue;
 
-                        for_each_engine(engine, dev_priv)
+                        for_each_engine(engine, dev_priv, id)
                                 ctx->engine[engine->id].initialised = false;
 
                         ctx->remap_slice = ALL_L3_SLICES(dev_priv);
                 }
 
-                for_each_engine(engine, dev_priv) {
+                for_each_engine(engine, dev_priv, id) {
                         struct intel_context *kce =
                                 &dev_priv->kernel_context->engine[engine->id];
 
@@ -563,6 +564,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
         struct drm_i915_private *dev_priv = req->i915;
         struct intel_ring *ring = req->ring;
         struct intel_engine_cs *engine = req->engine;
+        enum intel_engine_id id;
         u32 flags = hw_flags | MI_MM_SPACE_GTT;
         const int num_rings =
                 /* Use an extended w/a on ivb+ if signalling from other rings */
@@ -605,7 +607,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 
                         intel_ring_emit(ring,
                                         MI_LOAD_REGISTER_IMM(num_rings));
-                        for_each_engine(signaller, dev_priv) {
+                        for_each_engine(signaller, dev_priv, id) {
                                 if (signaller == engine)
                                         continue;
 
@@ -634,7 +636,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 
                         intel_ring_emit(ring,
                                         MI_LOAD_REGISTER_IMM(num_rings));
-                        for_each_engine(signaller, dev_priv) {
+                        for_each_engine(signaller, dev_priv, id) {
                                 if (signaller == engine)
                                         continue;
 
@@ -929,8 +931,9 @@ int i915_switch_context(struct drm_i915_gem_request *req)
 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 struct drm_i915_gem_request *req;
                 int ret;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 5b6f81c1dbca..b5e9e669f50f 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -37,8 +37,9 @@ static bool
 gpu_is_idle(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 if (intel_engine_is_active(engine))
                         return false;
         }
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 33c85227643d..e52affdcc125 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -370,8 +370,7 @@ static void reloc_cache_fini(struct reloc_cache *cache)
 
                         ggtt->base.clear_range(&ggtt->base,
                                                cache->node.start,
-                                               cache->node.size,
-                                               true);
+                                               cache->node.size);
                         drm_mm_remove_node(&cache->node);
                 } else {
                         i915_vma_unpin((struct i915_vma *)cache->node.mm);
@@ -429,7 +428,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
         }
 
         if (cache->vaddr) {
-                io_mapping_unmap_atomic(unmask_page(cache->vaddr));
+                io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
         } else {
                 struct i915_vma *vma;
                 int ret;
@@ -451,8 +450,8 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
                                  0, ggtt->mappable_end,
                                  DRM_MM_SEARCH_DEFAULT,
                                  DRM_MM_CREATE_DEFAULT);
-                        if (ret)
-                                return ERR_PTR(ret);
+                        if (ret) /* no inactive aperture space, use cpu reloc */
+                                return NULL;
                 } else {
                         ret = i915_vma_put_fence(vma);
                         if (ret) {
@@ -474,7 +473,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
                 offset += page << PAGE_SHIFT;
         }
 
-        vaddr = io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
+        vaddr = (void __force *) io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
         cache->page = page;
         cache->vaddr = (unsigned long)vaddr;
 
@@ -552,27 +551,13 @@ repeat:
         return 0;
 }
 
-static bool object_is_idle(struct drm_i915_gem_object *obj)
-{
-        unsigned long active = i915_gem_object_get_active(obj);
-        int idx;
-
-        for_each_active(active, idx) {
-                if (!i915_gem_active_is_idle(&obj->last_read[idx],
-                                             &obj->base.dev->struct_mutex))
-                        return false;
-        }
-
-        return true;
-}
-
 static int
 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
                                    struct eb_vmas *eb,
                                    struct drm_i915_gem_relocation_entry *reloc,
                                    struct reloc_cache *cache)
 {
-        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         struct drm_gem_object *target_obj;
         struct drm_i915_gem_object *target_i915_obj;
         struct i915_vma *target_vma;
@@ -591,7 +576,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
         /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
          * pipe_control writes because the gpu doesn't properly redirect them
          * through the ppgtt for non_secure batchbuffers. */
-        if (unlikely(IS_GEN6(dev) &&
+        if (unlikely(IS_GEN6(dev_priv) &&
             reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
                 ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
                                     PIN_GLOBAL);
@@ -649,10 +634,6 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
                 return -EINVAL;
         }
 
-        /* We can't wait for rendering with pagefaults disabled */
-        if (pagefault_disabled() && !object_is_idle(obj))
-                return -EFAULT;
-
         ret = relocate_entry(obj, reloc, cache, target_offset);
         if (ret)
                 return ret;
@@ -679,12 +660,23 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
         remain = entry->relocation_count;
         while (remain) {
                 struct drm_i915_gem_relocation_entry *r = stack_reloc;
-                int count = remain;
-                if (count > ARRAY_SIZE(stack_reloc))
-                        count = ARRAY_SIZE(stack_reloc);
+                unsigned long unwritten;
+                unsigned int count;
+
+                count = min_t(unsigned int, remain, ARRAY_SIZE(stack_reloc));
                 remain -= count;
 
-                if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]))) {
+                /* This is the fast path and we cannot handle a pagefault
+                 * whilst holding the struct mutex lest the user pass in the
+                 * relocations contained within a mmaped bo. For in such a case
+                 * we, the page fault handler would call i915_gem_fault() and
+                 * we would try to acquire the struct mutex again. Obviously
+                 * this is bad and so lockdep complains vehemently.
+                 */
+                pagefault_disable();
+                unwritten = __copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]));
+                pagefault_enable();
+                if (unlikely(unwritten)) {
                         ret = -EFAULT;
                         goto out;
                 }
@@ -696,11 +688,26 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
                         if (ret)
                                 goto out;
 
-                        if (r->presumed_offset != offset &&
-                            __put_user(r->presumed_offset,
-                                       &user_relocs->presumed_offset)) {
-                                ret = -EFAULT;
-                                goto out;
+                        if (r->presumed_offset != offset) {
+                                pagefault_disable();
+                                unwritten = __put_user(r->presumed_offset,
+                                                       &user_relocs->presumed_offset);
+                                pagefault_enable();
+                                if (unlikely(unwritten)) {
+                                        /* Note that reporting an error now
+                                         * leaves everything in an inconsistent
+                                         * state as we have *already* changed
+                                         * the relocation value inside the
+                                         * object. As we have not changed the
+                                         * reloc.presumed_offset or will not
+                                         * change the execobject.offset, on the
+                                         * call we may not rewrite the value
+                                         * inside the object, leaving it
+                                         * dangling and causing a GPU hang.
+                                         */
+                                        ret = -EFAULT;
+                                        goto out;
+                                }
                         }
 
                         user_relocs++;
@@ -740,20 +747,11 @@ i915_gem_execbuffer_relocate(struct eb_vmas *eb)
         struct i915_vma *vma;
         int ret = 0;
 
-        /* This is the fast path and we cannot handle a pagefault whilst
-         * holding the struct mutex lest the user pass in the relocations
-         * contained within a mmaped bo. For in such a case we, the page
-         * fault handler would call i915_gem_fault() and we would try to
-         * acquire the struct mutex again. Obviously this is bad and so
-         * lockdep complains vehemently.
-         */
-        pagefault_disable();
         list_for_each_entry(vma, &eb->vmas, exec_list) {
                 ret = i915_gem_execbuffer_relocate_vma(vma, eb);
                 if (ret)
                         break;
         }
-        pagefault_enable();
 
         return ret;
 }
@@ -1253,7 +1251,7 @@ validate_exec_list(struct drm_device *dev,
                         return -EFAULT;
 
                 if (likely(!i915.prefault_disable)) {
-                        if (fault_in_multipages_readable(ptr, length))
+                        if (fault_in_pages_readable(ptr, length))
                                 return -EFAULT;
                 }
         }
@@ -1599,12 +1597,12 @@ eb_select_engine(struct drm_i915_private *dev_priv,
                         return NULL;
                 }
 
-                engine = &dev_priv->engine[_VCS(bsd_idx)];
+                engine = dev_priv->engine[_VCS(bsd_idx)];
         } else {
-                engine = &dev_priv->engine[user_ring_map[user_ring_id]];
+                engine = dev_priv->engine[user_ring_map[user_ring_id]];
         }
 
-        if (!intel_engine_initialized(engine)) {
+        if (!engine) {
                 DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
                 return NULL;
         }
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c
index 8df1fa7234e8..a6daf2deab74 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence.c
@@ -290,6 +290,8 @@ i915_vma_put_fence(struct i915_vma *vma)
 {
         struct drm_i915_fence_reg *fence = vma->fence;
 
+        assert_rpm_wakelock_held(to_i915(vma->vm->dev));
+
         if (!fence)
                 return 0;
 
@@ -341,6 +343,8 @@ i915_vma_get_fence(struct i915_vma *vma)
         struct drm_i915_fence_reg *fence;
         struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL;
 
+        assert_rpm_wakelock_held(to_i915(vma->vm->dev));
+
         /* Just update our place in the LRU if our fence is getting reused. */
         if (vma->fence) {
                 fence = vma->fence;
@@ -371,6 +375,12 @@ void i915_gem_restore_fences(struct drm_device *dev)
         struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
+        /* Note that this may be called outside of struct_mutex, by
+         * runtime suspend/resume. The barrier we require is enforced by
+         * rpm itself - all access to fences/GTT are only within an rpm
+         * wakeref, and to acquire that wakeref you must pass through here.
+         */
+
         for (i = 0; i < dev_priv->num_fence_regs; i++) {
                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
                 struct i915_vma *vma = reg->vma;
@@ -379,10 +389,17 @@ void i915_gem_restore_fences(struct drm_device *dev)
                  * Commit delayed tiling changes if we have an object still
                  * attached to the fence, otherwise just clear the fence.
                  */
-                if (vma && !i915_gem_object_is_tiled(vma->obj))
+                if (vma && !i915_gem_object_is_tiled(vma->obj)) {
+                        GEM_BUG_ON(!reg->dirty);
+                        GEM_BUG_ON(vma->obj->fault_mappable);
+
+                        list_move(&reg->link, &dev_priv->mm.fence_list);
+                        vma->fence = NULL;
                         vma = NULL;
+                }
 
-                fence_update(reg, vma);
+                fence_write(reg, vma);
+                reg->vma = vma;
         }
 }
 
@@ -448,7 +465,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
         uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
         uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
 
-        if (INTEL_INFO(dev)->gen >= 8 || IS_VALLEYVIEW(dev)) {
+        if (INTEL_GEN(dev_priv) >= 8 || IS_VALLEYVIEW(dev_priv)) {
                 /*
                  * On BDW+, swizzling is not used. We leave the CPU memory
                  * controller in charge of optimizing memory accesses without
@@ -487,19 +504,20 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
                                 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
                         }
                 }
-        } else if (IS_GEN5(dev)) {
+        } else if (IS_GEN5(dev_priv)) {
                 /* On Ironlake whatever DRAM config, GPU always do
                  * same swizzling setup.
                  */
                 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
                 swizzle_y = I915_BIT_6_SWIZZLE_9;
-        } else if (IS_GEN2(dev)) {
+        } else if (IS_GEN2(dev_priv)) {
                 /* As far as we know, the 865 doesn't have these bit 6
                  * swizzling issues.
                  */
                 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
                 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
-        } else if (IS_MOBILE(dev) || (IS_GEN3(dev) && !IS_G33(dev))) {
+        } else if (IS_MOBILE(dev_priv) || (IS_GEN3(dev_priv) &&
+                   !IS_G33(dev_priv))) {
                 uint32_t dcc;
 
                 /* On 9xx chipsets, channel interleave by the CPU is
@@ -537,7 +555,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
                 }
 
                 /* check for L-shaped memory aka modified enhanced addressing */
-                if (IS_GEN4(dev) &&
+                if (IS_GEN4(dev_priv) &&
                     !(I915_READ(DCC2) & DCC2_MODIFIED_ENHANCED_DISABLE)) {
                         swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
                         swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 0bb4232f66bc..062fb0ad75da 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -191,15 +191,13 @@ static void ppgtt_unbind_vma(struct i915_vma *vma)
 {
         vma->vm->clear_range(vma->vm,
                              vma->node.start,
-                             vma->size,
-                             true);
+                             vma->size);
 }
 
 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
-                                  enum i915_cache_level level,
-                                  bool valid)
+                                  enum i915_cache_level level)
 {
-        gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
+        gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
         pte |= addr;
 
         switch (level) {
@@ -234,9 +232,9 @@ static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 
 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
                                  enum i915_cache_level level,
-                                 bool valid, u32 unused)
+                                 u32 unused)
 {
-        gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+        gen6_pte_t pte = GEN6_PTE_VALID;
         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
         switch (level) {
@@ -256,9 +254,9 @@ static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
                                  enum i915_cache_level level,
-                                 bool valid, u32 unused)
+                                 u32 unused)
 {
-        gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+        gen6_pte_t pte = GEN6_PTE_VALID;
         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
         switch (level) {
@@ -280,9 +278,9 @@ static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
                                  enum i915_cache_level level,
-                                 bool valid, u32 flags)
+                                 u32 flags)
 {
-        gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+        gen6_pte_t pte = GEN6_PTE_VALID;
         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
         if (!(flags & PTE_READ_ONLY))
@@ -296,9 +294,9 @@ static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
                                  enum i915_cache_level level,
-                                 bool valid, u32 unused)
+                                 u32 unused)
 {
-        gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+        gen6_pte_t pte = GEN6_PTE_VALID;
         pte |= HSW_PTE_ADDR_ENCODE(addr);
 
         if (level != I915_CACHE_NONE)
@@ -309,9 +307,9 @@ static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
                                   enum i915_cache_level level,
-                                  bool valid, u32 unused)
+                                  u32 unused)
 {
-        gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+        gen6_pte_t pte = GEN6_PTE_VALID;
         pte |= HSW_PTE_ADDR_ENCODE(addr);
 
         switch (level) {
@@ -373,27 +371,29 @@ static void *kmap_page_dma(struct i915_page_dma *p)
 /* We use the flushing unmap only with ppgtt structures:
  * page directories, page tables and scratch pages.
  */
-static void kunmap_page_dma(struct drm_device *dev, void *vaddr)
+static void kunmap_page_dma(struct drm_i915_private *dev_priv, void *vaddr)
 {
         /* There are only few exceptions for gen >=6. chv and bxt.
          * And we are not sure about the latter so play safe for now.
          */
-        if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
+        if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 
         kunmap_atomic(vaddr);
 }
 
 #define kmap_px(px) kmap_page_dma(px_base(px))
-#define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr))
+#define kunmap_px(ppgtt, vaddr) \
+                kunmap_page_dma(to_i915((ppgtt)->base.dev), (vaddr))
 
 #define setup_px(dev, px) setup_page_dma((dev), px_base(px))
 #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px))
-#define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v))
-#define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v))
+#define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v))
+#define fill32_px(dev_priv, px, v) \
+                fill_page_dma_32((dev_priv), px_base(px), (v))
 
-static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
-                          const uint64_t val)
+static void fill_page_dma(struct drm_i915_private *dev_priv,
+                          struct i915_page_dma *p, const uint64_t val)
 {
         int i;
         uint64_t * const vaddr = kmap_page_dma(p);
@@ -401,17 +401,17 @@ static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
         for (i = 0; i < 512; i++)
                 vaddr[i] = val;
 
-        kunmap_page_dma(dev, vaddr);
+        kunmap_page_dma(dev_priv, vaddr);
 }
 
-static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
-                             const uint32_t val32)
+static void fill_page_dma_32(struct drm_i915_private *dev_priv,
+                             struct i915_page_dma *p, const uint32_t val32)
 {
         uint64_t v = val32;
 
         v = v << 32 | val32;
 
-        fill_page_dma(dev, p, v);
+        fill_page_dma(dev_priv, p, v);
 }
 
 static int
@@ -472,9 +472,9 @@ static void gen8_initialize_pt(struct i915_address_space *vm,
         gen8_pte_t scratch_pte;
 
         scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-                                      I915_CACHE_LLC, true);
+                                      I915_CACHE_LLC);
 
-        fill_px(vm->dev, pt, scratch_pte);
+        fill_px(to_i915(vm->dev), pt, scratch_pte);
 }
 
 static void gen6_initialize_pt(struct i915_address_space *vm,
@@ -485,9 +485,9 @@ static void gen6_initialize_pt(struct i915_address_space *vm,
         WARN_ON(vm->scratch_page.daddr == 0);
 
         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-                                     I915_CACHE_LLC, true, 0);
+                                     I915_CACHE_LLC, 0);
 
-        fill32_px(vm->dev, pt, scratch_pte);
+        fill32_px(to_i915(vm->dev), pt, scratch_pte);
 }
 
 static struct i915_page_directory *alloc_pd(struct drm_device *dev)
@@ -534,7 +534,7 @@ static void gen8_initialize_pd(struct i915_address_space *vm,
 
         scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
 
-        fill_px(vm->dev, pd, scratch_pde);
+        fill_px(to_i915(vm->dev), pd, scratch_pde);
 }
 
 static int __pdp_init(struct drm_device *dev,
@@ -615,7 +615,7 @@ static void gen8_initialize_pdp(struct i915_address_space *vm,
 
         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 
-        fill_px(vm->dev, pdp, scratch_pdpe);
+        fill_px(to_i915(vm->dev), pdp, scratch_pdpe);
 }
 
 static void gen8_initialize_pml4(struct i915_address_space *vm,
@@ -626,7 +626,7 @@ static void gen8_initialize_pml4(struct i915_address_space *vm,
         scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
                                           I915_CACHE_LLC);
 
-        fill_px(vm->dev, pml4, scratch_pml4e);
+        fill_px(to_i915(vm->dev), pml4, scratch_pml4e);
 }
 
 static void
@@ -706,85 +706,157 @@ static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
         return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
 }
 
-static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm,
-                                       struct i915_page_directory_pointer *pdp,
-                                       uint64_t start,
-                                       uint64_t length,
-                                       gen8_pte_t scratch_pte)
+/* Removes entries from a single page table, releasing it if it's empty.
+ * Caller can use the return value to update higher-level entries.
+ */
+static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
+                                struct i915_page_table *pt,
+                                uint64_t start,
+                                uint64_t length)
 {
         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+        unsigned int pte_start = gen8_pte_index(start);
+        unsigned int num_entries = gen8_pte_count(start, length);
+        uint64_t pte;
         gen8_pte_t *pt_vaddr;
-        unsigned pdpe = gen8_pdpe_index(start);
-        unsigned pde = gen8_pde_index(start);
-        unsigned pte = gen8_pte_index(start);
-        unsigned num_entries = length >> PAGE_SHIFT;
-        unsigned last_pte, i;
+        gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
+                                                 I915_CACHE_LLC);
 
-        if (WARN_ON(!pdp))
-                return;
+        if (WARN_ON(!px_page(pt)))
+                return false;
 
-        while (num_entries) {
-                struct i915_page_directory *pd;
-                struct i915_page_table *pt;
+        bitmap_clear(pt->used_ptes, pte_start, num_entries);
 
-                if (WARN_ON(!pdp->page_directory[pdpe]))
-                        break;
+        if (bitmap_empty(pt->used_ptes, GEN8_PTES)) {
+                free_pt(vm->dev, pt);
+                return true;
+        }
 
-                pd = pdp->page_directory[pdpe];
+        pt_vaddr = kmap_px(pt);
 
-                if (WARN_ON(!pd->page_table[pde]))
-                        break;
+        for (pte = pte_start; pte < num_entries; pte++)
+                pt_vaddr[pte] = scratch_pte;
 
-                pt = pd->page_table[pde];
+        kunmap_px(ppgtt, pt_vaddr);
 
-                if (WARN_ON(!px_page(pt)))
-                        break;
+        return false;
+}
 
-                last_pte = pte + num_entries;
-                if (last_pte > GEN8_PTES)
-                        last_pte = GEN8_PTES;
+/* Removes entries from a single page dir, releasing it if it's empty.
+ * Caller can use the return value to update higher-level entries
+ */
+static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
+                                struct i915_page_directory *pd,
+                                uint64_t start,
+                                uint64_t length)
+{
+        struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+        struct i915_page_table *pt;
+        uint64_t pde;
+        gen8_pde_t *pde_vaddr;
+        gen8_pde_t scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt),
+                                                 I915_CACHE_LLC);
 
-                pt_vaddr = kmap_px(pt);
+        gen8_for_each_pde(pt, pd, start, length, pde) {
+                if (WARN_ON(!pd->page_table[pde]))
+                        break;
 
-                for (i = pte; i < last_pte; i++) {
-                        pt_vaddr[i] = scratch_pte;
-                        num_entries--;
+                if (gen8_ppgtt_clear_pt(vm, pt, start, length)) {
+                        __clear_bit(pde, pd->used_pdes);
+                        pde_vaddr = kmap_px(pd);
+                        pde_vaddr[pde] = scratch_pde;
+                        kunmap_px(ppgtt, pde_vaddr);
                 }
+        }
 
-                kunmap_px(ppgtt, pt_vaddr);
+        if (bitmap_empty(pd->used_pdes, I915_PDES)) {
+                free_pd(vm->dev, pd);
+                return true;
+        }
+
+        return false;
+}
 
-                pte = 0;
-                if (++pde == I915_PDES) {
-                        if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
-                                break;
-                        pde = 0;
+/* Removes entries from a single page dir pointer, releasing it if it's empty.
+ * Caller can use the return value to update higher-level entries
+ */
+static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
+                                 struct i915_page_directory_pointer *pdp,
+                                 uint64_t start,
+                                 uint64_t length)
+{
+        struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+        struct i915_page_directory *pd;
+        uint64_t pdpe;
+        gen8_ppgtt_pdpe_t *pdpe_vaddr;
+        gen8_ppgtt_pdpe_t scratch_pdpe =
+                gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
+
+        gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
+                if (WARN_ON(!pdp->page_directory[pdpe]))
+                        break;
+
+                if (gen8_ppgtt_clear_pd(vm, pd, start, length)) {
+                        __clear_bit(pdpe, pdp->used_pdpes);
+                        if (USES_FULL_48BIT_PPGTT(vm->dev)) {
+                                pdpe_vaddr = kmap_px(pdp);
+                                pdpe_vaddr[pdpe] = scratch_pdpe;
+                                kunmap_px(ppgtt, pdpe_vaddr);
+                        }
                 }
         }
+
+        if (USES_FULL_48BIT_PPGTT(vm->dev) &&
+            bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(vm->dev))) {
+                free_pdp(vm->dev, pdp);
+                return true;
+        }
+
+        return false;
 }
 
-static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
-                                   uint64_t start,
-                                   uint64_t length,
-                                   bool use_scratch)
+/* Removes entries from a single pml4.
+ * This is the top-level structure in 4-level page tables used on gen8+.
+ * Empty entries are always scratch pml4e.
+ */
+static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm,
+                                  struct i915_pml4 *pml4,
+                                  uint64_t start,
+                                  uint64_t length)
 {
         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-        gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-                                                 I915_CACHE_LLC, use_scratch);
+        struct i915_page_directory_pointer *pdp;
+        uint64_t pml4e;
+        gen8_ppgtt_pml4e_t *pml4e_vaddr;
+        gen8_ppgtt_pml4e_t scratch_pml4e =
+                gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC);
 
-        if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
-                gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length,
-                                           scratch_pte);
-        } else {
-                uint64_t pml4e;
-                struct i915_page_directory_pointer *pdp;
+        GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(vm->dev));
 
-                gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) {
-                        gen8_ppgtt_clear_pte_range(vm, pdp, start, length,
-                                                   scratch_pte);
+        gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
+                if (WARN_ON(!pml4->pdps[pml4e]))
+                        break;
+
+                if (gen8_ppgtt_clear_pdp(vm, pdp, start, length)) {
+                        __clear_bit(pml4e, pml4->used_pml4es);
+                        pml4e_vaddr = kmap_px(pml4);
+                        pml4e_vaddr[pml4e] = scratch_pml4e;
+                        kunmap_px(ppgtt, pml4e_vaddr);
                 }
         }
 }
 
+static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
+                                   uint64_t start, uint64_t length)
+{
+        struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+
+        if (USES_FULL_48BIT_PPGTT(vm->dev))
+                gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length);
+        else
+                gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length);
+}
+
 static void
 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
                               struct i915_page_directory_pointer *pdp,
@@ -809,7 +881,7 @@ gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
 
                 pt_vaddr[pte] =
                         gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
-                                        cache_level, true);
+                                        cache_level);
                 if (++pte == GEN8_PTES) {
                         kunmap_px(ppgtt, pt_vaddr);
                         pt_vaddr = NULL;
@@ -1452,7 +1524,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
         uint64_t start = ppgtt->base.start;
         uint64_t length = ppgtt->base.total;
         gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-                                                 I915_CACHE_LLC, true);
+                                                 I915_CACHE_LLC);
 
         if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
                 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
@@ -1569,7 +1641,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
         uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
 
         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-                                     I915_CACHE_LLC, true, 0);
+                                     I915_CACHE_LLC, 0);
 
         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
                 u32 expected;
@@ -1728,8 +1800,9 @@ static void gen8_ppgtt_enable(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0;
                 I915_WRITE(RING_MODE_GEN7(engine),
                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
@@ -1741,12 +1814,13 @@ static void gen7_ppgtt_enable(struct drm_device *dev)
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         uint32_t ecochk, ecobits;
+        enum intel_engine_id id;
 
         ecobits = I915_READ(GAC_ECO_BITS);
         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 
         ecochk = I915_READ(GAM_ECOCHK);
-        if (IS_HASWELL(dev)) {
+        if (IS_HASWELL(dev_priv)) {
                 ecochk |= ECOCHK_PPGTT_WB_HSW;
         } else {
                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
@@ -1754,7 +1828,7 @@ static void gen7_ppgtt_enable(struct drm_device *dev)
         }
         I915_WRITE(GAM_ECOCHK, ecochk);
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 /* GFX_MODE is per-ring on gen7+ */
                 I915_WRITE(RING_MODE_GEN7(engine),
                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
@@ -1782,8 +1856,7 @@ static void gen6_ppgtt_enable(struct drm_device *dev)
 /* PPGTT support for Sandybdrige/Gen6 and later */
 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
                                    uint64_t start,
-                                   uint64_t length,
-                                   bool use_scratch)
+                                   uint64_t length)
 {
         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
         gen6_pte_t *pt_vaddr, scratch_pte;
@@ -1794,7 +1867,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
         unsigned last_pte, i;
 
         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-                                     I915_CACHE_LLC, true, 0);
+                                     I915_CACHE_LLC, 0);
 
         while (num_entries) {
                 last_pte = first_pte + num_entries;
@@ -1832,7 +1905,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
                         pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
 
                 pt_vaddr[act_pte] =
-                        vm->pte_encode(addr, cache_level, true, flags);
+                        vm->pte_encode(addr, cache_level, flags);
 
                 if (++act_pte == GEN6_PTES) {
                         kunmap_px(ppgtt, pt_vaddr);
@@ -2056,11 +2129,11 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
         int ret;
 
         ppgtt->base.pte_encode = ggtt->base.pte_encode;
-        if (intel_vgpu_active(dev_priv) || IS_GEN6(dev))
+        if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
                 ppgtt->switch_mm = gen6_mm_switch;
-        else if (IS_HASWELL(dev))
+        else if (IS_HASWELL(dev_priv))
                 ppgtt->switch_mm = hsw_mm_switch;
-        else if (IS_GEN7(dev))
+        else if (IS_GEN7(dev_priv))
                 ppgtt->switch_mm = gen7_mm_switch;
         else
                 BUG();
@@ -2129,13 +2202,13 @@ static void gtt_write_workarounds(struct drm_device *dev)
          * workarounds here even if they get overwritten by GPU reset.
          */
         /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt */
-        if (IS_BROADWELL(dev))
+        if (IS_BROADWELL(dev_priv))
                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
-        else if (IS_CHERRYVIEW(dev))
+        else if (IS_CHERRYVIEW(dev_priv))
                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
-        else if (IS_SKYLAKE(dev))
+        else if (IS_SKYLAKE(dev_priv))
                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
 }
 
@@ -2157,6 +2230,8 @@ static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
 
 int i915_ppgtt_init_hw(struct drm_device *dev)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
         gtt_write_workarounds(dev);
 
         /* In the case of execlists, PPGTT is enabled by the context descriptor
@@ -2168,9 +2243,9 @@ int i915_ppgtt_init_hw(struct drm_device *dev)
         if (!USES_PPGTT(dev))
                 return 0;
 
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev_priv))
                 gen6_ppgtt_enable(dev);
-        else if (IS_GEN7(dev))
+        else if (IS_GEN7(dev_priv))
                 gen7_ppgtt_enable(dev);
         else if (INTEL_INFO(dev)->gen >= 8)
                 gen8_ppgtt_enable(dev);
@@ -2239,11 +2314,12 @@ static bool needs_idle_maps(struct drm_i915_private *dev_priv)
 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
         if (INTEL_INFO(dev_priv)->gen < 6)
                 return;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 u32 fault_reg;
                 fault_reg = I915_READ(RING_FAULT_REG(engine));
                 if (fault_reg & RING_FAULT_VALID) {
@@ -2260,7 +2336,10 @@ void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
                                    fault_reg & ~RING_FAULT_VALID);
                 }
         }
-        POSTING_READ(RING_FAULT_REG(&dev_priv->engine[RCS]));
+
+        /* Engine specific init may not have been done till this point. */
+        if (dev_priv->engine[RCS])
+                POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
 }
 
 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
@@ -2286,8 +2365,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
 
         i915_check_and_clear_faults(dev_priv);
 
-        ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total,
-                             true);
+        ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
 
         i915_ggtt_flush(dev_priv);
 }
@@ -2321,7 +2399,7 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 
         rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
-        gen8_set_pte(pte, gen8_pte_encode(addr, level, true));
+        gen8_set_pte(pte, gen8_pte_encode(addr, level));
 
         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
         POSTING_READ(GFX_FLSH_CNTL_GEN6);
@@ -2348,7 +2426,7 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
 
         for_each_sgt_dma(addr, sgt_iter, st) {
-                gtt_entry = gen8_pte_encode(addr, level, true);
+                gtt_entry = gen8_pte_encode(addr, level);
                 gen8_set_pte(&gtt_entries[i++], gtt_entry);
         }
 
@@ -2412,7 +2490,7 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 
         rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
-        iowrite32(vm->pte_encode(addr, level, true, flags), pte);
+        iowrite32(vm->pte_encode(addr, level, flags), pte);
 
         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
         POSTING_READ(GFX_FLSH_CNTL_GEN6);
@@ -2445,7 +2523,7 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
         gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
 
         for_each_sgt_dma(addr, sgt_iter, st) {
-                gtt_entry = vm->pte_encode(addr, level, true, flags);
+                gtt_entry = vm->pte_encode(addr, level, flags);
                 iowrite32(gtt_entry, &gtt_entries[i++]);
         }
 
@@ -2469,16 +2547,12 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 }
 
 static void nop_clear_range(struct i915_address_space *vm,
-                            uint64_t start,
-                            uint64_t length,
-                            bool use_scratch)
+                            uint64_t start, uint64_t length)
 {
 }
 
 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
-                                  uint64_t start,
-                                  uint64_t length,
-                                  bool use_scratch)
+                                  uint64_t start, uint64_t length)
 {
         struct drm_i915_private *dev_priv = to_i915(vm->dev);
         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
@@ -2498,8 +2572,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
                 num_entries = max_entries;
 
         scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-                                      I915_CACHE_LLC,
-                                      use_scratch);
+                                      I915_CACHE_LLC);
         for (i = 0; i < num_entries; i++)
                 gen8_set_pte(&gtt_base[i], scratch_pte);
         readl(gtt_base);
@@ -2509,8 +2582,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 
 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
                                   uint64_t start,
-                                  uint64_t length,
-                                  bool use_scratch)
+                                  uint64_t length)
 {
         struct drm_i915_private *dev_priv = to_i915(vm->dev);
         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
@@ -2530,7 +2602,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
                 num_entries = max_entries;
 
         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-                                     I915_CACHE_LLC, use_scratch, 0);
+                                     I915_CACHE_LLC, 0);
 
         for (i = 0; i < num_entries; i++)
                 iowrite32(scratch_pte, &gtt_base[i]);
@@ -2577,8 +2649,7 @@ static void i915_ggtt_insert_entries(struct i915_address_space *vm,
 
 static void i915_ggtt_clear_range(struct i915_address_space *vm,
                                   uint64_t start,
-                                  uint64_t length,
-                                  bool unused)
+                                  uint64_t length)
 {
         struct drm_i915_private *dev_priv = to_i915(vm->dev);
         unsigned first_entry = start >> PAGE_SHIFT;
@@ -2662,13 +2733,11 @@ static void ggtt_unbind_vma(struct i915_vma *vma)
 
         if (vma->flags & I915_VMA_GLOBAL_BIND)
                 vma->vm->clear_range(vma->vm,
-                                     vma->node.start, size,
-                                     true);
+                                     vma->node.start, size);
 
         if (vma->flags & I915_VMA_LOCAL_BIND && appgtt)
                 appgtt->base.clear_range(&appgtt->base,
-                                         vma->node.start, size,
-                                         true);
+                                         vma->node.start, size);
 }
 
 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
@@ -2717,6 +2786,7 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
          */
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         unsigned long hole_start, hole_end;
+        struct i915_hw_ppgtt *ppgtt;
         struct drm_mm_node *entry;
         int ret;
 
@@ -2724,45 +2794,48 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
         if (ret)
                 return ret;
 
+        /* Reserve a mappable slot for our lockless error capture */
+        ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm,
+                                                  &ggtt->error_capture,
+                                                  4096, 0, -1,
+                                                  0, ggtt->mappable_end,
+                                                  0, 0);
+        if (ret)
+                return ret;
+
         /* Clear any non-preallocated blocks */
         drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
                               hole_start, hole_end);
                 ggtt->base.clear_range(&ggtt->base, hole_start,
-                                     hole_end - hole_start, true);
+                                       hole_end - hole_start);
         }
 
         /* And finally clear the reserved guard page */
         ggtt->base.clear_range(&ggtt->base,
-                               ggtt->base.total - PAGE_SIZE, PAGE_SIZE,
-                               true);
+                               ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
 
         if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
-                struct i915_hw_ppgtt *ppgtt;
-
                 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
-                if (!ppgtt)
-                        return -ENOMEM;
+                if (!ppgtt) {
+                        ret = -ENOMEM;
+                        goto err;
+                }
 
                 ret = __hw_ppgtt_init(ppgtt, dev_priv);
-                if (ret) {
-                        kfree(ppgtt);
-                        return ret;
-                }
+                if (ret)
+                        goto err_ppgtt;
 
-                if (ppgtt->base.allocate_va_range)
+                if (ppgtt->base.allocate_va_range) {
                         ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
                                                             ppgtt->base.total);
-                if (ret) {
-                        ppgtt->base.cleanup(&ppgtt->base);
-                        kfree(ppgtt);
-                        return ret;
+                        if (ret)
+                                goto err_ppgtt_cleanup;
                 }
 
                 ppgtt->base.clear_range(&ppgtt->base,
                                         ppgtt->base.start,
-                                        ppgtt->base.total,
-                                        true);
+                                        ppgtt->base.total);
 
                 dev_priv->mm.aliasing_ppgtt = ppgtt;
                 WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma);
@@ -2770,6 +2843,14 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
         }
 
         return 0;
+
+err_ppgtt_cleanup:
+        ppgtt->base.cleanup(&ppgtt->base);
+err_ppgtt:
+        kfree(ppgtt);
+err:
+        drm_mm_remove_node(&ggtt->error_capture);
+        return ret;
 }
 
 /**
@@ -2788,6 +2869,9 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
 
         i915_gem_cleanup_stolen(&dev_priv->drm);
 
+        if (drm_mm_node_allocated(&ggtt->error_capture))
+                drm_mm_remove_node(&ggtt->error_capture);
+
         if (drm_mm_initialized(&ggtt->base.mm)) {
                 intel_vgt_deballoon(dev_priv);
 
@@ -2895,7 +2979,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
          * resort to an uncached mapping. The WC issue is easily caught by the
          * readback check when writing GTT PTE entries.
          */
-        if (IS_BROXTON(ggtt->base.dev))
+        if (IS_BROXTON(to_i915(ggtt->base.dev)))
                 ggtt->gsm = ioremap_nocache(phys_addr, size);
         else
                 ggtt->gsm = ioremap_wc(phys_addr, size);
@@ -3237,8 +3321,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
         i915_check_and_clear_faults(dev_priv);
 
         /* First fill our portion of the GTT with scratch pages */
-        ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total,
-                               true);
+        ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
 
         ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
 
@@ -3267,7 +3350,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
         ggtt->base.closed = false;
 
         if (INTEL_INFO(dev)->gen >= 8) {
-                if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
+                if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
                         chv_setup_private_ppat(dev_priv);
                 else
                         bdw_setup_private_ppat(dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index ec78be2f8c77..c241d8143255 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -395,7 +395,7 @@ struct i915_address_space {
         /* FIXME: Need a more generic return type */
         gen6_pte_t (*pte_encode)(dma_addr_t addr,
                                  enum i915_cache_level level,
-                                 bool valid, u32 flags); /* Create a valid PTE */
+                                 u32 flags); /* Create a valid PTE */
         /* flags for pte_encode */
 #define PTE_READ_ONLY   (1<<0)
         int (*allocate_va_range)(struct i915_address_space *vm,
@@ -403,8 +403,7 @@ struct i915_address_space {
                                  uint64_t length);
         void (*clear_range)(struct i915_address_space *vm,
                             uint64_t start,
-                            uint64_t length,
-                            bool use_scratch);
+                            uint64_t length);
         void (*insert_page)(struct i915_address_space *vm,
                             dma_addr_t addr,
                             uint64_t offset,
@@ -450,6 +449,8 @@ struct i915_ggtt {
         bool do_idle_maps;
 
         int mtrr;
+
+        struct drm_mm_node error_capture;
 };
 
 struct i915_hw_ppgtt {
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 95b7e9afd5f8..a98c0f42badd 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -72,9 +72,9 @@ render_state_get_rodata(const struct drm_i915_gem_request *req)
 
 static int render_state_setup(struct render_state *so)
 {
-        struct drm_device *dev = so->vma->vm->dev;
+        struct drm_i915_private *dev_priv = to_i915(so->vma->vm->dev);
         const struct intel_renderstate_rodata *rodata = so->rodata;
-        const bool has_64bit_reloc = INTEL_GEN(dev) >= 8;
+        const bool has_64bit_reloc = INTEL_GEN(dev_priv) >= 8;
         unsigned int i = 0, reloc_index = 0;
         struct page *page;
         u32 *d;
@@ -115,7 +115,7 @@ static int render_state_setup(struct render_state *so)
 
         so->aux_batch_offset = i * sizeof(u32);
 
-        if (HAS_POOLED_EU(dev)) {
+        if (HAS_POOLED_EU(dev_priv)) {
                 /*
                  * We always program 3x6 pool config but depending upon which
                  * subslice is disabled HW drops down to appropriate config
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index 40978bc12ceb..74ede1f53372 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -256,10 +256,11 @@ static int i915_gem_check_wedge(struct drm_i915_private *dev_priv)
 static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int ret;
 
         /* Carefully retire all requests without writing to the rings */
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 ret = intel_engine_idle(engine,
                                         I915_WAIT_INTERRUPTIBLE |
                                         I915_WAIT_LOCKED);
@@ -276,7 +277,7 @@ static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
         }
 
         /* Finally reset hw state */
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 intel_engine_init_seqno(engine, seqno);
 
         return 0;
@@ -328,6 +329,7 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 
         switch (state) {
         case FENCE_COMPLETE:
+                request->engine->last_submitted_seqno = request->fence.seqno;
                 request->engine->submit_request(request);
                 break;
 
@@ -641,8 +643,8 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
                                              &request->submitq);
 
         request->emitted_jiffies = jiffies;
-        request->previous_seqno = engine->last_submitted_seqno;
-        engine->last_submitted_seqno = request->fence.seqno;
+        request->previous_seqno = engine->last_pending_seqno;
+        engine->last_pending_seqno = request->fence.seqno;
         i915_gem_active_set(&engine->last_request, request);
         list_add_tail(&request->link, &engine->request_list);
         list_add_tail(&request->ring_link, &ring->request_list);
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 1c237d02f30b..de25b6e0a101 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -182,8 +182,9 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
                             !is_vmalloc_addr(obj->mapping))
                                 continue;
 
-                        if ((flags & I915_SHRINK_ACTIVE) == 0 &&
-                            i915_gem_object_is_active(obj))
+                        if (!(flags & I915_SHRINK_ACTIVE) &&
+                            (i915_gem_object_is_active(obj) ||
+                             obj->framebuffer_references))
                                 continue;
 
                         if (!can_release_pages(obj))
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 59989e8ee5dc..f4f6d3a48b05 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -115,7 +115,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
                 pci_read_config_dword(pdev, INTEL_BSM, &bsm);
 
                 base = bsm & INTEL_BSM_MASK;
-        } else if (IS_I865G(dev)) {
+        } else if (IS_I865G(dev_priv)) {
                 u32 tseg_size = 0;
                 u16 toud = 0;
                 u8 tmp;
@@ -154,7 +154,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
                 tom = tmp * MB(32);
 
                 base = tom - tseg_size - ggtt->stolen_size;
-        } else if (IS_845G(dev)) {
+        } else if (IS_845G(dev_priv)) {
                 u32 tseg_size = 0;
                 u32 tom;
                 u8 tmp;
@@ -178,7 +178,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
                 tom = tmp * MB(32);
 
                 base = tom - tseg_size - ggtt->stolen_size;
-        } else if (IS_I830(dev)) {
+        } else if (IS_I830(dev_priv)) {
                 u32 tseg_size = 0;
                 u32 tom;
                 u8 tmp;
@@ -204,7 +204,8 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
                 return 0;
 
         /* make sure we don't clobber the GTT if it's within stolen memory */
-        if (INTEL_INFO(dev)->gen <= 4 && !IS_G33(dev) && !IS_G4X(dev)) {
+        if (INTEL_GEN(dev_priv) <= 4 && !IS_G33(dev_priv) &&
+            !IS_G4X(dev_priv)) {
                 struct {
                         u32 start, end;
                 } stolen[2] = {
@@ -214,7 +215,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
                 u64 ggtt_start, ggtt_end;
 
                 ggtt_start = I915_READ(PGTBL_CTL);
-                if (IS_GEN4(dev))
+                if (IS_GEN4(dev_priv))
                         ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
                                      (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
                 else
@@ -270,7 +271,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
                  * GEN3 firmware likes to smash pci bridges into the stolen
                  * range. Apparently this works.
                  */
-                if (r == NULL && !IS_GEN3(dev)) {
+                if (r == NULL && !IS_GEN3(dev_priv)) {
                         DRM_ERROR("conflict detected with stolen region: [0x%08x - 0x%08x]\n",
                                   base, base + (uint32_t)ggtt->stolen_size);
                         base = 0;
@@ -437,7 +438,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
         case 3:
                 break;
         case 4:
-                if (IS_G4X(dev))
+                if (IS_G4X(dev_priv))
                         g4x_get_stolen_reserved(dev_priv, &reserved_base,
                                                 &reserved_size);
                 break;
@@ -456,7 +457,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
                 break;
         default:
                 if (IS_BROADWELL(dev_priv) ||
-                    IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev))
+                    IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
                         bdw_get_stolen_reserved(dev_priv, &reserved_base,
                                                 &reserved_size);
                 else
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index a14b1e3d4c78..c21bc0068d20 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -62,6 +62,7 @@
 static bool
 i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int tile_width;
 
         /* Linear is always fine */
@@ -71,8 +72,8 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
         if (tiling_mode > I915_TILING_LAST)
                 return false;
 
-        if (IS_GEN2(dev) ||
-            (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
+        if (IS_GEN2(dev_priv) ||
+            (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev_priv)))
                 tile_width = 128;
         else
                 tile_width = 512;
@@ -90,7 +91,7 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
                 if (stride > 8192)
                         return false;
 
-                if (IS_GEN3(dev)) {
+                if (IS_GEN3(dev_priv)) {
                         if (size > I830_FENCE_MAX_SIZE_VAL << 20)
                                 return false;
                 } else {
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index e537930c64b5..c6f780f5abc9 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -508,6 +508,10 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
         pvec = drm_malloc_gfp(npages, sizeof(struct page *), GFP_TEMPORARY);
         if (pvec != NULL) {
                 struct mm_struct *mm = obj->userptr.mm->mm;
+                unsigned int flags = 0;
+
+                if (!obj->userptr.read_only)
+                        flags |= FOLL_WRITE;
 
                 ret = -EFAULT;
                 if (atomic_inc_not_zero(&mm->mm_users)) {
@@ -517,7 +521,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
                                         (work->task, mm,
                                          obj->userptr.ptr + pinned * PAGE_SIZE,
                                          npages - pinned,
-                                         !obj->userptr.read_only, 0,
+                                         flags,
                                          pvec + pinned, NULL);
                                 if (ret < 0)
                                         break;
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 334f15df7c8d..242b9a927899 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -28,6 +28,8 @@
  */
 
 #include <generated/utsrelease.h>
+#include <linux/stop_machine.h>
+#include <linux/zlib.h>
 #include "i915_drv.h"
 
 static const char *engine_str(int engine)
@@ -172,6 +174,110 @@ static void i915_error_puts(struct drm_i915_error_state_buf *e,
 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
 #define err_puts(e, s) i915_error_puts(e, s)
 
+#ifdef CONFIG_DRM_I915_COMPRESS_ERROR
+
+static bool compress_init(struct z_stream_s *zstream)
+{
+        memset(zstream, 0, sizeof(*zstream));
+
+        zstream->workspace =
+                kmalloc(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
+                        GFP_ATOMIC | __GFP_NOWARN);
+        if (!zstream->workspace)
+                return false;
+
+        if (zlib_deflateInit(zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
+                kfree(zstream->workspace);
+                return false;
+        }
+
+        return true;
+}
+
+static int compress_page(struct z_stream_s *zstream,
+                         void *src,
+                         struct drm_i915_error_object *dst)
+{
+        zstream->next_in = src;
+        zstream->avail_in = PAGE_SIZE;
+
+        do {
+                if (zstream->avail_out == 0) {
+                        unsigned long page;
+
+                        page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
+                        if (!page)
+                                return -ENOMEM;
+
+                        dst->pages[dst->page_count++] = (void *)page;
+
+                        zstream->next_out = (void *)page;
+                        zstream->avail_out = PAGE_SIZE;
+                }
+
+                if (zlib_deflate(zstream, Z_SYNC_FLUSH) != Z_OK)
+                        return -EIO;
+        } while (zstream->avail_in);
+
+        /* Fallback to uncompressed if we increase size? */
+        if (0 && zstream->total_out > zstream->total_in)
+                return -E2BIG;
+
+        return 0;
+}
+
+static void compress_fini(struct z_stream_s *zstream,
+                          struct drm_i915_error_object *dst)
+{
+        if (dst) {
+                zlib_deflate(zstream, Z_FINISH);
+                dst->unused = zstream->avail_out;
+        }
+
+        zlib_deflateEnd(zstream);
+        kfree(zstream->workspace);
+}
+
+static void err_compression_marker(struct drm_i915_error_state_buf *m)
+{
+        err_puts(m, ":");
+}
+
+#else
+
+static bool compress_init(struct z_stream_s *zstream)
+{
+        return true;
+}
+
+static int compress_page(struct z_stream_s *zstream,
+                         void *src,
+                         struct drm_i915_error_object *dst)
+{
+        unsigned long page;
+
+        page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
+        if (!page)
+                return -ENOMEM;
+
+        dst->pages[dst->page_count++] =
+                memcpy((void *)page, src, PAGE_SIZE);
+
+        return 0;
+}
+
+static void compress_fini(struct z_stream_s *zstream,
+                          struct drm_i915_error_object *dst)
+{
+}
+
+static void err_compression_marker(struct drm_i915_error_state_buf *m)
+{
+        err_puts(m, "~");
+}
+
+#endif
+
 static void print_error_buffers(struct drm_i915_error_state_buf *m,
                                 const char *name,
                                 struct drm_i915_error_buffer *err,
@@ -228,13 +334,57 @@ static const char *hangcheck_action_to_str(enum intel_engine_hangcheck_action a)
         return "unknown";
 }
 
+static void error_print_instdone(struct drm_i915_error_state_buf *m,
+                                 struct drm_i915_error_engine *ee)
+{
+        int slice;
+        int subslice;
+
+        err_printf(m, "  INSTDONE: 0x%08x\n",
+                   ee->instdone.instdone);
+
+        if (ee->engine_id != RCS || INTEL_GEN(m->i915) <= 3)
+                return;
+
+        err_printf(m, "  SC_INSTDONE: 0x%08x\n",
+                   ee->instdone.slice_common);
+
+        if (INTEL_GEN(m->i915) <= 6)
+                return;
+
+        for_each_instdone_slice_subslice(m->i915, slice, subslice)
+                err_printf(m, "  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
+                           slice, subslice,
+                           ee->instdone.sampler[slice][subslice]);
+
+        for_each_instdone_slice_subslice(m->i915, slice, subslice)
+                err_printf(m, "  ROW_INSTDONE[%d][%d]: 0x%08x\n",
+                           slice, subslice,
+                           ee->instdone.row[slice][subslice]);
+}
+
+static void error_print_request(struct drm_i915_error_state_buf *m,
+                                const char *prefix,
+                                struct drm_i915_error_request *erq)
+{
+        if (!erq->seqno)
+                return;
+
+        err_printf(m, "%s pid %d, seqno %8x:%08x, emitted %dms ago, head %08x, tail %08x\n",
+                   prefix, erq->pid,
+                   erq->context, erq->seqno,
+                   jiffies_to_msecs(jiffies - erq->jiffies),
+                   erq->head, erq->tail);
+}
+
 static void error_print_engine(struct drm_i915_error_state_buf *m,
                                struct drm_i915_error_engine *ee)
 {
         err_printf(m, "%s command stream:\n", engine_str(ee->engine_id));
         err_printf(m, "  START: 0x%08x\n", ee->start);
-        err_printf(m, "  HEAD:  0x%08x\n", ee->head);
-        err_printf(m, "  TAIL:  0x%08x\n", ee->tail);
+        err_printf(m, "  HEAD:  0x%08x [0x%08x]\n", ee->head, ee->rq_head);
+        err_printf(m, "  TAIL:  0x%08x [0x%08x, 0x%08x]\n",
+                   ee->tail, ee->rq_post, ee->rq_tail);
         err_printf(m, "  CTL:   0x%08x\n", ee->ctl);
         err_printf(m, "  MODE:  0x%08x\n", ee->mode);
         err_printf(m, "  HWS:   0x%08x\n", ee->hws);
@@ -242,7 +392,9 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
                    (u32)(ee->acthd>>32), (u32)ee->acthd);
         err_printf(m, "  IPEIR: 0x%08x\n", ee->ipeir);
         err_printf(m, "  IPEHR: 0x%08x\n", ee->ipehr);
-        err_printf(m, "  INSTDONE: 0x%08x\n", ee->instdone);
+
+        error_print_instdone(m, ee);
+
         if (ee->batchbuffer) {
                 u64 start = ee->batchbuffer->gtt_offset;
                 u64 end = start + ee->batchbuffer->gtt_size;
@@ -296,6 +448,8 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
         err_printf(m, "  hangcheck: %s [%d]\n",
                    hangcheck_action_to_str(ee->hangcheck_action),
                    ee->hangcheck_score);
+        error_print_request(m, "  ELSP[0]: ", &ee->execlist[0]);
+        error_print_request(m, "  ELSP[1]: ", &ee->execlist[1]);
 }
 
 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
@@ -307,28 +461,72 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
         va_end(args);
 }
 
+static int
+ascii85_encode_len(int len)
+{
+        return DIV_ROUND_UP(len, 4);
+}
+
+static bool
+ascii85_encode(u32 in, char *out)
+{
+        int i;
+
+        if (in == 0)
+                return false;
+
+        out[5] = '\0';
+        for (i = 5; i--; ) {
+                out[i] = '!' + in % 85;
+                in /= 85;
+        }
+
+        return true;
+}
+
 static void print_error_obj(struct drm_i915_error_state_buf *m,
+                            struct intel_engine_cs *engine,
+                            const char *name,
                             struct drm_i915_error_object *obj)
 {
-        int page, offset, elt;
+        char out[6];
+        int page;
+
+        if (!obj)
+                return;
+
+        if (name) {
+                err_printf(m, "%s --- %s = 0x%08x %08x\n",
+                           engine ? engine->name : "global", name,
+                           upper_32_bits(obj->gtt_offset),
+                           lower_32_bits(obj->gtt_offset));
+        }
+
+        err_compression_marker(m);
+        for (page = 0; page < obj->page_count; page++) {
+                int i, len;
+
+                len = PAGE_SIZE;
+                if (page == obj->page_count - 1)
+                        len -= obj->unused;
+                len = ascii85_encode_len(len);
 
-        for (page = offset = 0; page < obj->page_count; page++) {
-                for (elt = 0; elt < PAGE_SIZE/4; elt++) {
-                        err_printf(m, "%08x :  %08x\n", offset,
-                                   obj->pages[page][elt]);
-                        offset += 4;
+                for (i = 0; i < len; i++) {
+                        if (ascii85_encode(obj->pages[page][i], out))
+                                err_puts(m, out);
+                        else
+                                err_puts(m, "z");
                 }
         }
+        err_puts(m, "\n");
 }
 
 static void err_print_capabilities(struct drm_i915_error_state_buf *m,
                                    const struct intel_device_info *info)
 {
 #define PRINT_FLAG(x)  err_printf(m, #x ": %s\n", yesno(info->x))
-#define SEP_SEMICOLON ;
-        DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_SEMICOLON);
+        DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
-#undef SEP_SEMICOLON
 }
 
 int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
@@ -339,8 +537,8 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
         struct pci_dev *pdev = dev_priv->drm.pdev;
         struct drm_i915_error_state *error = error_priv->error;
         struct drm_i915_error_object *obj;
-        int i, j, offset, elt;
         int max_hangcheck_score;
+        int i, j;
 
         if (!error) {
                 err_printf(m, "no error state collected\n");
@@ -391,7 +589,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                 for (i = 0; i < 4; i++)
                         err_printf(m, "GTIER gt %d: 0x%08x\n", i,
                                    error->gtier[i]);
-        } else if (HAS_PCH_SPLIT(dev) || IS_VALLEYVIEW(dev))
+        } else if (HAS_PCH_SPLIT(dev_priv) || IS_VALLEYVIEW(dev_priv))
                 err_printf(m, "GTIER: 0x%08x\n", error->gtier[0]);
         err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
         err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
@@ -402,10 +600,6 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
         for (i = 0; i < dev_priv->num_fence_regs; i++)
                 err_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
 
-        for (i = 0; i < ARRAY_SIZE(error->extra_instdone); i++)
-                err_printf(m, "  INSTDONE_%d: 0x%08x\n", i,
-                           error->extra_instdone[i]);
-
         if (INTEL_INFO(dev)->gen >= 6) {
                 err_printf(m, "ERROR: 0x%08x\n", error->error);
 
@@ -416,7 +610,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                 err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
         }
 
-        if (IS_GEN7(dev))
+        if (IS_GEN7(dev_priv))
                 err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
 
         for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
@@ -438,7 +632,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
                         len += scnprintf(buf + len, sizeof(buf), "%s%s",
                                          first ? "" : ", ",
-                                         dev_priv->engine[j].name);
+                                         dev_priv->engine[j]->name);
                         first = 0;
                 }
                 scnprintf(buf + len, sizeof(buf), ")");
@@ -456,7 +650,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
                 obj = ee->batchbuffer;
                 if (obj) {
-                        err_puts(m, dev_priv->engine[i].name);
+                        err_puts(m, dev_priv->engine[i]->name);
                         if (ee->pid != -1)
                                 err_printf(m, " (submitted by %s [%d])",
                                            ee->comm,
@@ -464,37 +658,23 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                         err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
                                    upper_32_bits(obj->gtt_offset),
                                    lower_32_bits(obj->gtt_offset));
-                        print_error_obj(m, obj);
-                }
-
-                obj = ee->wa_batchbuffer;
-                if (obj) {
-                        err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
-                                   dev_priv->engine[i].name,
-                                   lower_32_bits(obj->gtt_offset));
-                        print_error_obj(m, obj);
+                        print_error_obj(m, dev_priv->engine[i], NULL, obj);
                 }
 
                 if (ee->num_requests) {
                         err_printf(m, "%s --- %d requests\n",
-                                   dev_priv->engine[i].name,
+                                   dev_priv->engine[i]->name,
                                    ee->num_requests);
-                        for (j = 0; j < ee->num_requests; j++) {
-                                err_printf(m, "  pid %d, seqno 0x%08x, emitted %ld, head 0x%08x, tail 0x%08x\n",
-                                           ee->requests[j].pid,
-                                           ee->requests[j].seqno,
-                                           ee->requests[j].jiffies,
-                                           ee->requests[j].head,
-                                           ee->requests[j].tail);
-                        }
+                        for (j = 0; j < ee->num_requests; j++)
+                                error_print_request(m, " ", &ee->requests[j]);
                 }
 
                 if (IS_ERR(ee->waiters)) {
                         err_printf(m, "%s --- ? waiters [unable to acquire spinlock]\n",
-                                   dev_priv->engine[i].name);
+                                   dev_priv->engine[i]->name);
                 } else if (ee->num_waiters) {
                         err_printf(m, "%s --- %d waiters\n",
-                                   dev_priv->engine[i].name,
+                                   dev_priv->engine[i]->name,
                                    ee->num_waiters);
                         for (j = 0; j < ee->num_waiters; j++) {
                                 err_printf(m, " seqno 0x%08x for %s [%d]\n",
@@ -504,77 +684,23 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                         }
                 }
 
-                if ((obj = ee->ringbuffer)) {
-                        err_printf(m, "%s --- ringbuffer = 0x%08x\n",
-                                   dev_priv->engine[i].name,
-                                   lower_32_bits(obj->gtt_offset));
-                        print_error_obj(m, obj);
-                }
+                print_error_obj(m, dev_priv->engine[i],
+                                "ringbuffer", ee->ringbuffer);
 
-                if ((obj = ee->hws_page)) {
-                        u64 hws_offset = obj->gtt_offset;
-                        u32 *hws_page = &obj->pages[0][0];
+                print_error_obj(m, dev_priv->engine[i],
+                                "HW Status", ee->hws_page);
 
-                        if (i915.enable_execlists) {
-                                hws_offset += LRC_PPHWSP_PN * PAGE_SIZE;
-                                hws_page = &obj->pages[LRC_PPHWSP_PN][0];
-                        }
-                        err_printf(m, "%s --- HW Status = 0x%08llx\n",
-                                   dev_priv->engine[i].name, hws_offset);
-                        offset = 0;
-                        for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
-                                err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-                                           offset,
-                                           hws_page[elt],
-                                           hws_page[elt+1],
-                                           hws_page[elt+2],
-                                           hws_page[elt+3]);
-                                offset += 16;
-                        }
-                }
+                print_error_obj(m, dev_priv->engine[i],
+                                "HW context", ee->ctx);
 
-                obj = ee->wa_ctx;
-                if (obj) {
-                        u64 wa_ctx_offset = obj->gtt_offset;
-                        u32 *wa_ctx_page = &obj->pages[0][0];
-                        struct intel_engine_cs *engine = &dev_priv->engine[RCS];
-                        u32 wa_ctx_size = (engine->wa_ctx.indirect_ctx.size +
-                                           engine->wa_ctx.per_ctx.size);
-
-                        err_printf(m, "%s --- WA ctx batch buffer = 0x%08llx\n",
-                                   dev_priv->engine[i].name, wa_ctx_offset);
-                        offset = 0;
-                        for (elt = 0; elt < wa_ctx_size; elt += 4) {
-                                err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-                                           offset,
-                                           wa_ctx_page[elt + 0],
-                                           wa_ctx_page[elt + 1],
-                                           wa_ctx_page[elt + 2],
-                                           wa_ctx_page[elt + 3]);
-                                offset += 16;
-                        }
-                }
+                print_error_obj(m, dev_priv->engine[i],
+                                "WA context", ee->wa_ctx);
 
-                if ((obj = ee->ctx)) {
-                        err_printf(m, "%s --- HW Context = 0x%08x\n",
-                                   dev_priv->engine[i].name,
-                                   lower_32_bits(obj->gtt_offset));
-                        print_error_obj(m, obj);
-                }
+                print_error_obj(m, dev_priv->engine[i],
+                                "WA batchbuffer", ee->wa_batchbuffer);
         }
 
-        if ((obj = error->semaphore)) {
-                err_printf(m, "Semaphore page = 0x%08x\n",
-                           lower_32_bits(obj->gtt_offset));
-                for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
-                        err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-                                   elt * 4,
-                                   obj->pages[0][elt],
-                                   obj->pages[0][elt+1],
-                                   obj->pages[0][elt+2],
-                                   obj->pages[0][elt+3]);
-                }
-        }
+        print_error_obj(m, NULL, "Semaphores", error->semaphore);
 
         if (error->overlay)
                 intel_overlay_print_error_state(m, error->overlay);
@@ -629,7 +755,7 @@ static void i915_error_object_free(struct drm_i915_error_object *obj)
                 return;
 
         for (page = 0; page < obj->page_count; page++)
-                kfree(obj->pages[page]);
+                free_page((unsigned long)obj->pages[page]);
 
         kfree(obj);
 }
@@ -667,104 +793,63 @@ static void i915_error_state_free(struct kref *error_ref)
 }
 
 static struct drm_i915_error_object *
-i915_error_object_create(struct drm_i915_private *dev_priv,
+i915_error_object_create(struct drm_i915_private *i915,
                          struct i915_vma *vma)
 {
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-        struct drm_i915_gem_object *src;
+        struct i915_ggtt *ggtt = &i915->ggtt;
+        const u64 slot = ggtt->error_capture.start;
         struct drm_i915_error_object *dst;
-        int num_pages;
-        bool use_ggtt;
-        int i = 0;
-        u64 reloc_offset;
+        struct z_stream_s zstream;
+        unsigned long num_pages;
+        struct sgt_iter iter;
+        dma_addr_t dma;
 
         if (!vma)
                 return NULL;
 
-        src = vma->obj;
-        if (!src->pages)
-                return NULL;
-
-        num_pages = src->base.size >> PAGE_SHIFT;
-
-        dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
+        num_pages = min_t(u64, vma->size, vma->obj->base.size) >> PAGE_SHIFT;
+        num_pages = DIV_ROUND_UP(10 * num_pages, 8); /* worstcase zlib growth */
+        dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *),
+                      GFP_ATOMIC | __GFP_NOWARN);
         if (!dst)
                 return NULL;
 
         dst->gtt_offset = vma->node.start;
         dst->gtt_size = vma->node.size;
+        dst->page_count = 0;
+        dst->unused = 0;
 
-        reloc_offset = dst->gtt_offset;
-        use_ggtt = (src->cache_level == I915_CACHE_NONE &&
-                   (vma->flags & I915_VMA_GLOBAL_BIND) &&
-                   reloc_offset + num_pages * PAGE_SIZE <= ggtt->mappable_end);
-
-        /* Cannot access stolen address directly, try to use the aperture */
-        if (src->stolen) {
-                use_ggtt = true;
-
-                if (!(vma->flags & I915_VMA_GLOBAL_BIND))
-                        goto unwind;
-
-                reloc_offset = vma->node.start;
-                if (reloc_offset + num_pages * PAGE_SIZE > ggtt->mappable_end)
-                        goto unwind;
+        if (!compress_init(&zstream)) {
+                kfree(dst);
+                return NULL;
         }
 
-        /* Cannot access snooped pages through the aperture */
-        if (use_ggtt && src->cache_level != I915_CACHE_NONE &&
-            !HAS_LLC(dev_priv))
-                goto unwind;
+        for_each_sgt_dma(dma, iter, vma->pages) {
+                void __iomem *s;
+                int ret;
 
-        dst->page_count = num_pages;
-        while (num_pages--) {
-                unsigned long flags;
-                void *d;
+                ggtt->base.insert_page(&ggtt->base, dma, slot,
+                                       I915_CACHE_NONE, 0);
 
-                d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
-                if (d == NULL)
-                        goto unwind;
-
-                local_irq_save(flags);
-                if (use_ggtt) {
-                        void __iomem *s;
-
-                        /* Simply ignore tiling or any overlapping fence.
-                         * It's part of the error state, and this hopefully
-                         * captures what the GPU read.
-                         */
-
-                        s = io_mapping_map_atomic_wc(&ggtt->mappable,
-                                                     reloc_offset);
-                        memcpy_fromio(d, s, PAGE_SIZE);
-                        io_mapping_unmap_atomic(s);
-                } else {
-                        struct page *page;
-                        void *s;
-
-                        page = i915_gem_object_get_page(src, i);
-
-                        drm_clflush_pages(&page, 1);
-
-                        s = kmap_atomic(page);
-                        memcpy(d, s, PAGE_SIZE);
-                        kunmap_atomic(s);
-
-                        drm_clflush_pages(&page, 1);
-                }
-                local_irq_restore(flags);
+                s = io_mapping_map_atomic_wc(&ggtt->mappable, slot);
+                ret = compress_page(&zstream, (void  __force *)s, dst);
+                io_mapping_unmap_atomic(s);
 
-                dst->pages[i++] = d;
-                reloc_offset += PAGE_SIZE;
+                if (ret)
+                        goto unwind;
         }
-
-        return dst;
+        goto out;
 
 unwind:
-        while (i--)
-                kfree(dst->pages[i]);
+        while (dst->page_count--)
+                free_page((unsigned long)dst->pages[dst->page_count]);
         kfree(dst);
-        return NULL;
+        dst = NULL;
+
+out:
+        compress_fini(&zstream, dst);
+        ggtt->base.clear_range(&ggtt->base, slot, PAGE_SIZE);
+        return dst;
 }
 
 /* The error capture is special as tries to run underneath the normal
@@ -855,7 +940,8 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
                         if (engine_id)
                                 *engine_id = i;
 
-                        return error->engine[i].ipehr ^ error->engine[i].instdone;
+                        return error->engine[i].ipehr ^
+                               error->engine[i].instdone.instdone;
                 }
         }
 
@@ -891,7 +977,7 @@ static void gen8_record_semaphore_state(struct drm_i915_error_state *error,
         if (!error->semaphore)
                 return;
 
-        for_each_engine_id(to, dev_priv, id) {
+        for_each_engine(to, dev_priv, id) {
                 int idx;
                 u16 signal_offset;
                 u32 *tmp;
@@ -998,7 +1084,6 @@ static void error_record_engine_registers(struct drm_i915_error_state *error,
                 ee->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
                 ee->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
                 ee->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
-                ee->instdone = I915_READ(RING_INSTDONE(engine->mmio_base));
                 ee->instps = I915_READ(RING_INSTPS(engine->mmio_base));
                 ee->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
                 if (INTEL_GEN(dev_priv) >= 8) {
@@ -1010,9 +1095,10 @@ static void error_record_engine_registers(struct drm_i915_error_state *error,
                 ee->faddr = I915_READ(DMA_FADD_I8XX);
                 ee->ipeir = I915_READ(IPEIR);
                 ee->ipehr = I915_READ(IPEHR);
-                ee->instdone = I915_READ(GEN2_INSTDONE);
         }
 
+        intel_engine_get_instdone(engine, &ee->instdone);
+
         ee->waiting = intel_engine_has_waiter(engine);
         ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
         ee->acthd = intel_engine_get_active_head(engine);
@@ -1079,6 +1165,20 @@ static void error_record_engine_registers(struct drm_i915_error_state *error,
         }
 }
 
+static void record_request(struct drm_i915_gem_request *request,
+                           struct drm_i915_error_request *erq)
+{
+        erq->context = request->ctx->hw_id;
+        erq->seqno = request->fence.seqno;
+        erq->jiffies = request->emitted_jiffies;
+        erq->head = request->head;
+        erq->tail = request->tail;
+
+        rcu_read_lock();
+        erq->pid = request->ctx->pid ? pid_nr(request->ctx->pid) : 0;
+        rcu_read_unlock();
+}
+
 static void engine_record_requests(struct intel_engine_cs *engine,
                                    struct drm_i915_gem_request *first,
                                    struct drm_i915_error_engine *ee)
@@ -1102,8 +1202,6 @@ static void engine_record_requests(struct intel_engine_cs *engine,
         count = 0;
         request = first;
         list_for_each_entry_from(request, &engine->request_list, link) {
-                struct drm_i915_error_request *erq;
-
                 if (count >= ee->num_requests) {
                         /*
                          * If the ring request list was changed in
@@ -1123,19 +1221,22 @@ static void engine_record_requests(struct intel_engine_cs *engine,
                         break;
                 }
 
-                erq = &ee->requests[count++];
-                erq->seqno = request->fence.seqno;
-                erq->jiffies = request->emitted_jiffies;
-                erq->head = request->head;
-                erq->tail = request->tail;
-
-                rcu_read_lock();
-                erq->pid = request->ctx->pid ? pid_nr(request->ctx->pid) : 0;
-                rcu_read_unlock();
+                record_request(request, &ee->requests[count++]);
         }
         ee->num_requests = count;
 }
 
+static void error_record_engine_execlists(struct intel_engine_cs *engine,
+                                          struct drm_i915_error_engine *ee)
+{
+        unsigned int n;
+
+        for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
+                if (engine->execlist_port[n].request)
+                        record_request(engine->execlist_port[n].request,
+                                       &ee->execlist[n]);
+}
+
 static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
                                   struct drm_i915_error_state *error)
 {
@@ -1146,20 +1247,21 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
                 i915_error_object_create(dev_priv, dev_priv->semaphore);
 
         for (i = 0; i < I915_NUM_ENGINES; i++) {
-                struct intel_engine_cs *engine = &dev_priv->engine[i];
+                struct intel_engine_cs *engine = dev_priv->engine[i];
                 struct drm_i915_error_engine *ee = &error->engine[i];
                 struct drm_i915_gem_request *request;
 
                 ee->pid = -1;
                 ee->engine_id = -1;
 
-                if (!intel_engine_initialized(engine))
+                if (!engine)
                         continue;
 
                 ee->engine_id = i;
 
                 error_record_engine_registers(error, engine, ee);
                 error_record_engine_waiters(engine, ee);
+                error_record_engine_execlists(engine, ee);
 
                 request = i915_gem_find_active_request(engine);
                 if (request) {
@@ -1202,6 +1304,10 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
                         error->simulated |=
                                 request->ctx->flags & CONTEXT_NO_ERROR_CAPTURE;
 
+                        ee->rq_head = request->head;
+                        ee->rq_post = request->postfix;
+                        ee->rq_tail = request->tail;
+
                         ring = request->ring;
                         ee->cpu_ring_head = ring->head;
                         ee->cpu_ring_tail = ring->tail;
@@ -1318,13 +1424,13 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
          */
 
         /* 1: Registers specific to a single generation */
-        if (IS_VALLEYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv)) {
                 error->gtier[0] = I915_READ(GTIER);
                 error->ier = I915_READ(VLV_IER);
                 error->forcewake = I915_READ_FW(FORCEWAKE_VLV);
         }
 
-        if (IS_GEN7(dev))
+        if (IS_GEN7(dev_priv))
                 error->err_int = I915_READ(GEN7_ERR_INT);
 
         if (INTEL_INFO(dev)->gen >= 8) {
@@ -1332,7 +1438,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
                 error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
         }
 
-        if (IS_GEN6(dev)) {
+        if (IS_GEN6(dev_priv)) {
                 error->forcewake = I915_READ_FW(FORCEWAKE);
                 error->gab_ctl = I915_READ(GAB_CTL);
                 error->gfx_mode = I915_READ(GFX_MODE);
@@ -1349,7 +1455,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
         }
 
         /* 3: Feature specific registers */
-        if (IS_GEN6(dev) || IS_GEN7(dev)) {
+        if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
                 error->gam_ecochk = I915_READ(GAM_ECOCHK);
                 error->gac_eco = I915_READ(GAC_ECO_BITS);
         }
@@ -1362,18 +1468,16 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
                 error->ier = I915_READ(GEN8_DE_MISC_IER);
                 for (i = 0; i < 4; i++)
                         error->gtier[i] = I915_READ(GEN8_GT_IER(i));
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
                 error->ier = I915_READ(DEIER);
                 error->gtier[0] = I915_READ(GTIER);
-        } else if (IS_GEN2(dev)) {
+        } else if (IS_GEN2(dev_priv)) {
                 error->ier = I915_READ16(IER);
-        } else if (!IS_VALLEYVIEW(dev)) {
+        } else if (!IS_VALLEYVIEW(dev_priv)) {
                 error->ier = I915_READ(IER);
         }
         error->eir = I915_READ(EIR);
         error->pgtbl_er = I915_READ(PGTBL_ER);
-
-        i915_get_extra_instdone(dev_priv, error->extra_instdone);
 }
 
 static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
@@ -1418,6 +1522,27 @@ static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
                sizeof(error->device_info));
 }
 
+static int capture(void *data)
+{
+        struct drm_i915_error_state *error = data;
+
+        i915_capture_gen_state(error->i915, error);
+        i915_capture_reg_state(error->i915, error);
+        i915_gem_record_fences(error->i915, error);
+        i915_gem_record_rings(error->i915, error);
+        i915_capture_active_buffers(error->i915, error);
+        i915_capture_pinned_buffers(error->i915, error);
+
+        do_gettimeofday(&error->time);
+
+        error->overlay = intel_overlay_capture_error_state(error->i915);
+        error->display = intel_display_capture_error_state(error->i915);
+
+        return 0;
+}
+
+#define DAY_AS_SECONDS(x) (24 * 60 * 60 * (x))
+
 /**
  * i915_capture_error_state - capture an error record for later analysis
  * @dev: drm device
@@ -1435,6 +1560,9 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
         struct drm_i915_error_state *error;
         unsigned long flags;
 
+        if (!i915.error_capture)
+                return;
+
         if (READ_ONCE(dev_priv->gpu_error.first_error))
                 return;
 
@@ -1446,18 +1574,9 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
         }
 
         kref_init(&error->ref);
+        error->i915 = dev_priv;
 
-        i915_capture_gen_state(dev_priv, error);
-        i915_capture_reg_state(dev_priv, error);
-        i915_gem_record_fences(dev_priv, error);
-        i915_gem_record_rings(dev_priv, error);
-        i915_capture_active_buffers(dev_priv, error);
-        i915_capture_pinned_buffers(dev_priv, error);
-
-        do_gettimeofday(&error->time);
-
-        error->overlay = intel_overlay_capture_error_state(dev_priv);
-        error->display = intel_display_capture_error_state(dev_priv);
+        stop_machine(capture, error, NULL);
 
         i915_error_capture_msg(dev_priv, error, engine_mask, error_msg);
         DRM_INFO("%s\n", error->error_msg);
@@ -1476,7 +1595,8 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
                 return;
         }
 
-        if (!warned) {
+        if (!warned &&
+            ktime_get_real_seconds() - DRIVER_TIMESTAMP < DAY_AS_SECONDS(180)) {
                 DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
                 DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
                 DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
@@ -1497,7 +1617,6 @@ void i915_error_state_get(struct drm_device *dev,
         if (error_priv->error)
                 kref_get(&error_priv->error->ref);
         spin_unlock_irq(&dev_priv->gpu_error.lock);
-
 }
 
 void i915_error_state_put(struct i915_error_state_file_priv *error_priv)
@@ -1519,33 +1638,3 @@ void i915_destroy_error_state(struct drm_device *dev)
         if (error)
                 kref_put(&error->ref, i915_error_state_free);
 }
-
-const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
-{
-        switch (type) {
-        case I915_CACHE_NONE: return " uncached";
-        case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
-        case I915_CACHE_L3_LLC: return " L3+LLC";
-        case I915_CACHE_WT: return " WT";
-        default: return "";
-        }
-}
-
-/* NB: please notice the memset */
-void i915_get_extra_instdone(struct drm_i915_private *dev_priv,
-                             uint32_t *instdone)
-{
-        memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
-
-        if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv))
-                instdone[0] = I915_READ(GEN2_INSTDONE);
-        else if (IS_GEN4(dev_priv) || IS_GEN5(dev_priv) || IS_GEN6(dev_priv)) {
-                instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
-                instdone[1] = I915_READ(GEN4_INSTDONE1);
-        } else if (INTEL_GEN(dev_priv) >= 7) {
-                instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
-                instdone[1] = I915_READ(GEN7_SC_INSTDONE);
-                instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
-                instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
-        }
-}
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c
index 43358e18d34c..a1f76c8f8cde 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/i915_guc_submission.c
@@ -451,6 +451,18 @@ int i915_guc_wq_reserve(struct drm_i915_gem_request *request)
         return ret;
 }
 
+void i915_guc_wq_unreserve(struct drm_i915_gem_request *request)
+{
+        const size_t wqi_size = sizeof(struct guc_wq_item);
+        struct i915_guc_client *gc = request->i915->guc.execbuf_client;
+
+        GEM_BUG_ON(READ_ONCE(gc->wq_rsvd) < wqi_size);
+
+        spin_lock(&gc->wq_lock);
+        gc->wq_rsvd -= wqi_size;
+        spin_unlock(&gc->wq_lock);
+}
+
 /* Construct a Work Item and append it to the GuC's Work Queue */
 static void guc_wq_item_append(struct i915_guc_client *gc,
                                struct drm_i915_gem_request *rq)
@@ -905,6 +917,7 @@ static void guc_addon_create(struct intel_guc *guc)
         struct guc_policies *policies;
         struct guc_mmio_reg_state *reg_state;
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         struct page *page;
         u32 size;
 
@@ -932,10 +945,10 @@ static void guc_addon_create(struct intel_guc *guc)
          * so its address won't change after we've told the GuC where
          * to find it.
          */
-        engine = &dev_priv->engine[RCS];
+        engine = dev_priv->engine[RCS];
         ads->golden_context_lrca = engine->status_page.ggtt_offset;
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 ads->eng_state_size[engine->guc_id] = intel_lr_context_size(engine);
 
         /* GuC scheduling policies */
@@ -948,7 +961,7 @@ static void guc_addon_create(struct intel_guc *guc)
         /* MMIO reg state */
         reg_state = (void *)policies + sizeof(struct guc_policies);
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 reg_state->mmio_white_list[engine->guc_id].mmio_start =
                         engine->mmio_base + GUC_MMIO_WHITE_LIST_START;
 
@@ -1002,9 +1015,10 @@ int i915_guc_submission_init(struct drm_i915_private *dev_priv)
 int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
 {
         struct intel_guc *guc = &dev_priv->guc;
+        struct drm_i915_gem_request *request;
         struct i915_guc_client *client;
         struct intel_engine_cs *engine;
-        struct drm_i915_gem_request *request;
+        enum intel_engine_id id;
 
         /* client for execbuf submission */
         client = guc_client_alloc(dev_priv,
@@ -1021,7 +1035,7 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
         guc_init_doorbell_hw(guc);
 
         /* Take over from manual control of ELSP (execlists) */
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 engine->submit_request = i915_guc_submit;
 
                 /* Replay the current set of previously submitted requests */
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index c128fdbd24e4..23315e5461bf 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -350,6 +350,9 @@ void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
 
 void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
 {
+        if (READ_ONCE(dev_priv->rps.interrupts_enabled))
+                return;
+
         spin_lock_irq(&dev_priv->irq_lock);
         WARN_ON_ONCE(dev_priv->rps.pm_iir);
         WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
@@ -368,6 +371,9 @@ u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 
 void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
 {
+        if (!READ_ONCE(dev_priv->rps.interrupts_enabled))
+                return;
+
         spin_lock_irq(&dev_priv->irq_lock);
         dev_priv->rps.interrupts_enabled = false;
 
@@ -1052,8 +1058,9 @@ static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
 static bool any_waiters(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 if (intel_engine_has_waiter(engine))
                         return true;
 
@@ -1251,20 +1258,20 @@ static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
                                u32 gt_iir)
 {
         if (gt_iir & GT_RENDER_USER_INTERRUPT)
-                notify_ring(&dev_priv->engine[RCS]);
+                notify_ring(dev_priv->engine[RCS]);
         if (gt_iir & ILK_BSD_USER_INTERRUPT)
-                notify_ring(&dev_priv->engine[VCS]);
+                notify_ring(dev_priv->engine[VCS]);
 }
 
 static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
                                u32 gt_iir)
 {
         if (gt_iir & GT_RENDER_USER_INTERRUPT)
-                notify_ring(&dev_priv->engine[RCS]);
+                notify_ring(dev_priv->engine[RCS]);
         if (gt_iir & GT_BSD_USER_INTERRUPT)
-                notify_ring(&dev_priv->engine[VCS]);
+                notify_ring(dev_priv->engine[VCS]);
         if (gt_iir & GT_BLT_USER_INTERRUPT)
-                notify_ring(&dev_priv->engine[BCS]);
+                notify_ring(dev_priv->engine[BCS]);
 
         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
                       GT_BSD_CS_ERROR_INTERRUPT |
@@ -1334,21 +1341,21 @@ static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
                                 u32 gt_iir[4])
 {
         if (gt_iir[0]) {
-                gen8_cs_irq_handler(&dev_priv->engine[RCS],
+                gen8_cs_irq_handler(dev_priv->engine[RCS],
                                     gt_iir[0], GEN8_RCS_IRQ_SHIFT);
-                gen8_cs_irq_handler(&dev_priv->engine[BCS],
+                gen8_cs_irq_handler(dev_priv->engine[BCS],
                                     gt_iir[0], GEN8_BCS_IRQ_SHIFT);
         }
 
         if (gt_iir[1]) {
-                gen8_cs_irq_handler(&dev_priv->engine[VCS],
+                gen8_cs_irq_handler(dev_priv->engine[VCS],
                                     gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
-                gen8_cs_irq_handler(&dev_priv->engine[VCS2],
+                gen8_cs_irq_handler(dev_priv->engine[VCS2],
                                     gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
         }
 
         if (gt_iir[3])
-                gen8_cs_irq_handler(&dev_priv->engine[VECS],
+                gen8_cs_irq_handler(dev_priv->engine[VECS],
                                     gt_iir[3], GEN8_VECS_IRQ_SHIFT);
 
         if (gt_iir[2] & dev_priv->pm_rps_events)
@@ -1592,7 +1599,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 
         if (HAS_VEBOX(dev_priv)) {
                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
-                        notify_ring(&dev_priv->engine[VECS]);
+                        notify_ring(dev_priv->engine[VECS]);
 
                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
                         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
@@ -2545,92 +2552,52 @@ static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
         wake_up_all(&dev_priv->gpu_error.reset_queue);
 }
 
-static void i915_report_and_clear_eir(struct drm_i915_private *dev_priv)
+static inline void
+i915_err_print_instdone(struct drm_i915_private *dev_priv,
+                        struct intel_instdone *instdone)
 {
-        uint32_t instdone[I915_NUM_INSTDONE_REG];
-        u32 eir = I915_READ(EIR);
-        int pipe, i;
+        int slice;
+        int subslice;
+
+        pr_err("  INSTDONE: 0x%08x\n", instdone->instdone);
 
-        if (!eir)
+        if (INTEL_GEN(dev_priv) <= 3)
                 return;
 
-        pr_err("render error detected, EIR: 0x%08x\n", eir);
+        pr_err("  SC_INSTDONE: 0x%08x\n", instdone->slice_common);
 
-        i915_get_extra_instdone(dev_priv, instdone);
+        if (INTEL_GEN(dev_priv) <= 6)
+                return;
 
-        if (IS_G4X(dev_priv)) {
-                if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
-                        u32 ipeir = I915_READ(IPEIR_I965);
-
-                        pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
-                        pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
-                        for (i = 0; i < ARRAY_SIZE(instdone); i++)
-                                pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
-                        pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
-                        pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
-                        I915_WRITE(IPEIR_I965, ipeir);
-                        POSTING_READ(IPEIR_I965);
-                }
-                if (eir & GM45_ERROR_PAGE_TABLE) {
-                        u32 pgtbl_err = I915_READ(PGTBL_ER);
-                        pr_err("page table error\n");
-                        pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
-                        I915_WRITE(PGTBL_ER, pgtbl_err);
-                        POSTING_READ(PGTBL_ER);
-                }
-        }
+        for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+                pr_err("  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
+                       slice, subslice, instdone->sampler[slice][subslice]);
 
-        if (!IS_GEN2(dev_priv)) {
-                if (eir & I915_ERROR_PAGE_TABLE) {
-                        u32 pgtbl_err = I915_READ(PGTBL_ER);
-                        pr_err("page table error\n");
-                        pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
-                        I915_WRITE(PGTBL_ER, pgtbl_err);
-                        POSTING_READ(PGTBL_ER);
-                }
-        }
+        for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+                pr_err("  ROW_INSTDONE[%d][%d]: 0x%08x\n",
+                       slice, subslice, instdone->row[slice][subslice]);
+}
 
-        if (eir & I915_ERROR_MEMORY_REFRESH) {
-                pr_err("memory refresh error:\n");
-                for_each_pipe(dev_priv, pipe)
-                        pr_err("pipe %c stat: 0x%08x\n",
-                               pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
-                /* pipestat has already been acked */
-        }
-        if (eir & I915_ERROR_INSTRUCTION) {
-                pr_err("instruction error\n");
-                pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
-                for (i = 0; i < ARRAY_SIZE(instdone); i++)
-                        pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
-                if (INTEL_GEN(dev_priv) < 4) {
-                        u32 ipeir = I915_READ(IPEIR);
-
-                        pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
-                        pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
-                        pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
-                        I915_WRITE(IPEIR, ipeir);
-                        POSTING_READ(IPEIR);
-                } else {
-                        u32 ipeir = I915_READ(IPEIR_I965);
-
-                        pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
-                        pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
-                        pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
-                        pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
-                        I915_WRITE(IPEIR_I965, ipeir);
-                        POSTING_READ(IPEIR_I965);
-                }
-        }
+static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
+{
+        u32 eir;
 
-        I915_WRITE(EIR, eir);
-        POSTING_READ(EIR);
+        if (!IS_GEN2(dev_priv))
+                I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
+
+        if (INTEL_GEN(dev_priv) < 4)
+                I915_WRITE(IPEIR, I915_READ(IPEIR));
+        else
+                I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
+
+        I915_WRITE(EIR, I915_READ(EIR));
         eir = I915_READ(EIR);
         if (eir) {
                 /*
                  * some errors might have become stuck,
                  * mask them.
                  */
-                DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
+                DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
                 I915_WRITE(EMR, I915_READ(EMR) | eir);
                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
         }
@@ -2659,7 +2626,7 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
         va_end(args);
 
         i915_capture_error_state(dev_priv, engine_mask, error_msg);
-        i915_report_and_clear_eir(dev_priv);
+        i915_clear_error_registers(dev_priv);
 
         if (!engine_mask)
                 return;
@@ -2688,45 +2655,40 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
-static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
+static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        if (INTEL_INFO(dev)->gen >= 4)
-                i915_enable_pipestat(dev_priv, pipe,
-                                     PIPE_START_VBLANK_INTERRUPT_STATUS);
-        else
-                i915_enable_pipestat(dev_priv, pipe,
-                                     PIPE_VBLANK_INTERRUPT_STATUS);
+        i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         return 0;
 }
 
-static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
+static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
-        uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
-                                                     DE_PIPE_VBLANK(pipe);
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        ilk_enable_display_irq(dev_priv, bit);
+        i915_enable_pipestat(dev_priv, pipe,
+                             PIPE_START_VBLANK_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         return 0;
 }
 
-static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
+static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
+        uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
+                DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        i915_enable_pipestat(dev_priv, pipe,
-                             PIPE_START_VBLANK_INTERRUPT_STATUS);
+        ilk_enable_display_irq(dev_priv, bit);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         return 0;
@@ -2747,38 +2709,36 @@ static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
-static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
+static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        i915_disable_pipestat(dev_priv, pipe,
-                              PIPE_VBLANK_INTERRUPT_STATUS |
-                              PIPE_START_VBLANK_INTERRUPT_STATUS);
+        i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
+static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
-        uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
-                                                     DE_PIPE_VBLANK(pipe);
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        ilk_disable_display_irq(dev_priv, bit);
+        i915_disable_pipestat(dev_priv, pipe,
+                              PIPE_START_VBLANK_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
+static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
+        uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
+                DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        i915_disable_pipestat(dev_priv, pipe,
-                              PIPE_START_VBLANK_INTERRUPT_STATUS);
+        ilk_disable_display_irq(dev_priv, bit);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
@@ -2810,29 +2770,30 @@ semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
 {
         struct drm_i915_private *dev_priv = engine->i915;
         struct intel_engine_cs *signaller;
+        enum intel_engine_id id;
 
         if (INTEL_GEN(dev_priv) >= 8) {
-                for_each_engine(signaller, dev_priv) {
+                for_each_engine(signaller, dev_priv, id) {
                         if (engine == signaller)
                                 continue;
 
-                        if (offset == signaller->semaphore.signal_ggtt[engine->id])
+                        if (offset == signaller->semaphore.signal_ggtt[engine->hw_id])
                                 return signaller;
                 }
         } else {
                 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
 
-                for_each_engine(signaller, dev_priv) {
+                for_each_engine(signaller, dev_priv, id) {
                         if(engine == signaller)
                                 continue;
 
-                        if (sync_bits == signaller->semaphore.mbox.wait[engine->id])
+                        if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id])
                                 return signaller;
                 }
         }
 
-        DRM_DEBUG_DRIVER("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
-                         engine->id, ipehr, offset);
+        DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n",
+                         engine->name, ipehr, offset);
 
         return ERR_PTR(-ENODEV);
 }
@@ -2943,35 +2904,52 @@ static int semaphore_passed(struct intel_engine_cs *engine)
 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 engine->hangcheck.deadlock = 0;
 }
 
+static bool instdone_unchanged(u32 current_instdone, u32 *old_instdone)
+{
+        u32 tmp = current_instdone | *old_instdone;
+        bool unchanged;
+
+        unchanged = tmp == *old_instdone;
+        *old_instdone |= tmp;
+
+        return unchanged;
+}
+
 static bool subunits_stuck(struct intel_engine_cs *engine)
 {
-        u32 instdone[I915_NUM_INSTDONE_REG];
+        struct drm_i915_private *dev_priv = engine->i915;
+        struct intel_instdone instdone;
+        struct intel_instdone *accu_instdone = &engine->hangcheck.instdone;
         bool stuck;
-        int i;
+        int slice;
+        int subslice;
 
         if (engine->id != RCS)
                 return true;
 
-        i915_get_extra_instdone(engine->i915, instdone);
+        intel_engine_get_instdone(engine, &instdone);
 
         /* There might be unstable subunit states even when
          * actual head is not moving. Filter out the unstable ones by
          * accumulating the undone -> done transitions and only
          * consider those as progress.
          */
-        stuck = true;
-        for (i = 0; i < I915_NUM_INSTDONE_REG; i++) {
-                const u32 tmp = instdone[i] | engine->hangcheck.instdone[i];
-
-                if (tmp != engine->hangcheck.instdone[i])
-                        stuck = false;
-
-                engine->hangcheck.instdone[i] |= tmp;
+        stuck = instdone_unchanged(instdone.instdone,
+                                   &accu_instdone->instdone);
+        stuck &= instdone_unchanged(instdone.slice_common,
+                                    &accu_instdone->slice_common);
+
+        for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
+                stuck &= instdone_unchanged(instdone.sampler[slice][subslice],
+                                            &accu_instdone->sampler[slice][subslice]);
+                stuck &= instdone_unchanged(instdone.row[slice][subslice],
+                                            &accu_instdone->row[slice][subslice]);
         }
 
         return stuck;
@@ -2983,7 +2961,7 @@ head_stuck(struct intel_engine_cs *engine, u64 acthd)
         if (acthd != engine->hangcheck.acthd) {
 
                 /* Clear subunit states on head movement */
-                memset(engine->hangcheck.instdone, 0,
+                memset(&engine->hangcheck.instdone, 0,
                        sizeof(engine->hangcheck.instdone));
 
                 return HANGCHECK_ACTIVE;
@@ -3055,6 +3033,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
                 container_of(work, typeof(*dev_priv),
                              gpu_error.hangcheck_work.work);
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         unsigned int hung = 0, stuck = 0;
         int busy_count = 0;
 #define BUSY 1
@@ -3074,7 +3053,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
          */
         intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 bool busy = intel_engine_has_waiter(engine);
                 u64 acthd;
                 u32 seqno;
@@ -3153,7 +3132,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
                         /* Clear head and subunit states on seqno movement */
                         acthd = 0;
 
-                        memset(engine->hangcheck.instdone, 0,
+                        memset(&engine->hangcheck.instdone, 0,
                                sizeof(engine->hangcheck.instdone));
                 }
 
@@ -3191,12 +3170,12 @@ static void ibx_irq_reset(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (HAS_PCH_NOP(dev))
+        if (HAS_PCH_NOP(dev_priv))
                 return;
 
         GEN5_IRQ_RESET(SDE);
 
-        if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
+        if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
                 I915_WRITE(SERR_INT, 0xffffffff);
 }
 
@@ -3212,7 +3191,7 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (HAS_PCH_NOP(dev))
+        if (HAS_PCH_NOP(dev_priv))
                 return;
 
         WARN_ON(I915_READ(SDEIER) != 0);
@@ -3287,7 +3266,7 @@ static void ironlake_irq_reset(struct drm_device *dev)
         I915_WRITE(HWSTAM, 0xffffffff);
 
         GEN5_IRQ_RESET(DE);
-        if (IS_GEN7(dev))
+        if (IS_GEN7(dev_priv))
                 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
 
         gen5_gt_irq_reset(dev);
@@ -3337,7 +3316,7 @@ static void gen8_irq_reset(struct drm_device *dev)
         GEN5_IRQ_RESET(GEN8_DE_MISC_);
         GEN5_IRQ_RESET(GEN8_PCU_);
 
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 ibx_irq_reset(dev);
 }
 
@@ -3526,10 +3505,10 @@ static void ibx_irq_postinstall(struct drm_device *dev)
         struct drm_i915_private *dev_priv = to_i915(dev);
         u32 mask;
 
-        if (HAS_PCH_NOP(dev))
+        if (HAS_PCH_NOP(dev_priv))
                 return;
 
-        if (HAS_PCH_IBX(dev))
+        if (HAS_PCH_IBX(dev_priv))
                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
         else
                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
@@ -3546,14 +3525,14 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
         pm_irqs = gt_irqs = 0;
 
         dev_priv->gt_irq_mask = ~0;
-        if (HAS_L3_DPF(dev)) {
+        if (HAS_L3_DPF(dev_priv)) {
                 /* L3 parity interrupt is always unmasked. */
-                dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
-                gt_irqs |= GT_PARITY_ERROR(dev);
+                dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
+                gt_irqs |= GT_PARITY_ERROR(dev_priv);
         }
 
         gt_irqs |= GT_RENDER_USER_INTERRUPT;
-        if (IS_GEN5(dev)) {
+        if (IS_GEN5(dev_priv)) {
                 gt_irqs |= ILK_BSD_USER_INTERRUPT;
         } else {
                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
@@ -3610,7 +3589,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 
         ibx_irq_postinstall(dev);
 
-        if (IS_IRONLAKE_M(dev)) {
+        if (IS_IRONLAKE_M(dev_priv)) {
                 /* Enable PCU event interrupts
                  *
                  * spinlocking not required here for correctness since interrupt
@@ -3750,13 +3729,13 @@ static int gen8_irq_postinstall(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 ibx_irq_pre_postinstall(dev);
 
         gen8_gt_irq_postinstall(dev_priv);
         gen8_de_irq_postinstall(dev_priv);
 
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 ibx_irq_postinstall(dev);
 
         I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
@@ -3965,7 +3944,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                 new_iir = I915_READ16(IIR); /* Flush posted writes */
 
                 if (iir & I915_USER_INTERRUPT)
-                        notify_ring(&dev_priv->engine[RCS]);
+                        notify_ring(dev_priv->engine[RCS]);
 
                 for_each_pipe(dev_priv, pipe) {
                         int plane = pipe;
@@ -4162,7 +4141,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                 new_iir = I915_READ(IIR); /* Flush posted writes */
 
                 if (iir & I915_USER_INTERRUPT)
-                        notify_ring(&dev_priv->engine[RCS]);
+                        notify_ring(dev_priv->engine[RCS]);
 
                 for_each_pipe(dev_priv, pipe) {
                         int plane = pipe;
@@ -4394,9 +4373,9 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                 new_iir = I915_READ(IIR); /* Flush posted writes */
 
                 if (iir & I915_USER_INTERRUPT)
-                        notify_ring(&dev_priv->engine[RCS]);
+                        notify_ring(dev_priv->engine[RCS]);
                 if (iir & I915_BSD_USER_INTERRUPT)
-                        notify_ring(&dev_priv->engine[VCS]);
+                        notify_ring(dev_priv->engine[VCS]);
 
                 for_each_pipe(dev_priv, pipe) {
                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
@@ -4533,16 +4512,16 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
                 dev->driver->irq_preinstall = cherryview_irq_preinstall;
                 dev->driver->irq_postinstall = cherryview_irq_postinstall;
                 dev->driver->irq_uninstall = cherryview_irq_uninstall;
-                dev->driver->enable_vblank = valleyview_enable_vblank;
-                dev->driver->disable_vblank = valleyview_disable_vblank;
+                dev->driver->enable_vblank = i965_enable_vblank;
+                dev->driver->disable_vblank = i965_disable_vblank;
                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
         } else if (IS_VALLEYVIEW(dev_priv)) {
                 dev->driver->irq_handler = valleyview_irq_handler;
                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
-                dev->driver->enable_vblank = valleyview_enable_vblank;
-                dev->driver->disable_vblank = valleyview_disable_vblank;
+                dev->driver->enable_vblank = i965_enable_vblank;
+                dev->driver->disable_vblank = i965_disable_vblank;
                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
         } else if (INTEL_INFO(dev_priv)->gen >= 8) {
                 dev->driver->irq_handler = gen8_irq_handler;
@@ -4551,13 +4530,13 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
                 dev->driver->irq_uninstall = gen8_irq_uninstall;
                 dev->driver->enable_vblank = gen8_enable_vblank;
                 dev->driver->disable_vblank = gen8_disable_vblank;
-                if (IS_BROXTON(dev))
+                if (IS_BROXTON(dev_priv))
                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
-                else if (HAS_PCH_SPT(dev) || HAS_PCH_KBP(dev))
+                else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
                         dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
                 else
                         dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
                 dev->driver->irq_handler = ironlake_irq_handler;
                 dev->driver->irq_preinstall = ironlake_irq_reset;
                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
@@ -4571,21 +4550,25 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
                         dev->driver->irq_handler = i8xx_irq_handler;
                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
+                        dev->driver->enable_vblank = i8xx_enable_vblank;
+                        dev->driver->disable_vblank = i8xx_disable_vblank;
                 } else if (IS_GEN3(dev_priv)) {
                         dev->driver->irq_preinstall = i915_irq_preinstall;
                         dev->driver->irq_postinstall = i915_irq_postinstall;
                         dev->driver->irq_uninstall = i915_irq_uninstall;
                         dev->driver->irq_handler = i915_irq_handler;
+                        dev->driver->enable_vblank = i8xx_enable_vblank;
+                        dev->driver->disable_vblank = i8xx_disable_vblank;
                 } else {
                         dev->driver->irq_preinstall = i965_irq_preinstall;
                         dev->driver->irq_postinstall = i965_irq_postinstall;
                         dev->driver->irq_uninstall = i965_irq_uninstall;
                         dev->driver->irq_handler = i965_irq_handler;
+                        dev->driver->enable_vblank = i965_enable_vblank;
+                        dev->driver->disable_vblank = i965_disable_vblank;
                 }
                 if (I915_HAS_HOTPLUG(dev_priv))
                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
-                dev->driver->enable_vblank = i915_enable_vblank;
-                dev->driver->disable_vblank = i915_disable_vblank;
         }
 }
 
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 768ad89d9cd4..629e4334719c 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -47,6 +47,7 @@ struct i915_params i915 __read_mostly = {
         .load_detect_test = 0,
         .force_reset_modeset_test = 0,
         .reset = true,
+        .error_capture = true,
         .invert_brightness = 0,
         .disable_display = 0,
         .enable_cmd_parser = 1,
@@ -115,6 +116,14 @@ MODULE_PARM_DESC(vbt_sdvo_panel_type,
 module_param_named_unsafe(reset, i915.reset, bool, 0600);
 MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+module_param_named(error_capture, i915.error_capture, bool, 0600);
+MODULE_PARM_DESC(error_capture,
+        "Record the GPU state following a hang. "
+        "This information in /sys/class/drm/card<N>/error is vital for "
+        "triaging and debugging hangs.");
+#endif
+
 module_param_named_unsafe(enable_hangcheck, i915.enable_hangcheck, bool, 0644);
 MODULE_PARM_DESC(enable_hangcheck,
         "Periodically check GPU activity for detecting hangs. "
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 3a0dd78ddb38..94efc899c1ef 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -59,6 +59,7 @@ struct i915_params {
         bool load_detect_test;
         bool force_reset_modeset_test;
         bool reset;
+        bool error_capture;
         bool disable_display;
         bool verbose_state_checks;
         bool nuclear_pageflip;
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 687c768833b3..31e6edd08dd0 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -431,9 +431,6 @@ static const struct pci_device_id pciidlist[] = {
 };
 MODULE_DEVICE_TABLE(pci, pciidlist);
 
-extern int i915_driver_load(struct pci_dev *pdev,
-                            const struct pci_device_id *ent);
-
 static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
         struct intel_device_info *intel_info =
@@ -463,8 +460,6 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
         return i915_driver_load(pdev, ent);
 }
 
-extern void i915_driver_unload(struct drm_device *dev);
-
 static void i915_pci_remove(struct pci_dev *pdev)
 {
         struct drm_device *dev = pci_get_drvdata(pdev);
@@ -473,8 +468,6 @@ static void i915_pci_remove(struct pci_dev *pdev)
         drm_dev_unref(dev);
 }
 
-extern const struct dev_pm_ops i915_pm_ops;
-
 static struct pci_driver i915_pci_driver = {
         .name = DRIVER_NAME,
         .id_table = pciidlist,
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 70d96162def6..00efaa13974d 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -86,8 +86,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define DEVEN 0x54
 #define   DEVEN_MCHBAR_EN (1 << 28)
 
-#define BSM 0x5c
-#define   BSM_MASK (0xFFFF << 20)
+/* BSM in include/drm/i915_drm.h */
 
 #define HPLLCC  0xc0 /* 85x only */
 #define   GC_CLOCK_CONTROL_MASK         (0x7 << 0)
@@ -1605,6 +1604,7 @@ enum skl_disp_power_wells {
 #define RING_HEAD(base)         _MMIO((base)+0x34)
 #define RING_START(base)        _MMIO((base)+0x38)
 #define RING_CTL(base)          _MMIO((base)+0x3c)
+#define   RING_CTL_SIZE(size)   ((size) - PAGE_SIZE) /* in bytes -> pages */
 #define RING_SYNC_0(base)       _MMIO((base)+0x40)
 #define RING_SYNC_1(base)       _MMIO((base)+0x44)
 #define RING_SYNC_2(base)       _MMIO((base)+0x48)
@@ -1708,7 +1708,11 @@ enum skl_disp_power_wells {
 #define GEN7_SC_INSTDONE        _MMIO(0x7100)
 #define GEN7_SAMPLER_INSTDONE   _MMIO(0xe160)
 #define GEN7_ROW_INSTDONE       _MMIO(0xe164)
-#define I915_NUM_INSTDONE_REG   4
+#define GEN8_MCR_SELECTOR               _MMIO(0xfdc)
+#define   GEN8_MCR_SLICE(slice)         (((slice) & 3) << 26)
+#define   GEN8_MCR_SLICE_MASK           GEN8_MCR_SLICE(3)
+#define   GEN8_MCR_SUBSLICE(subslice)   (((subslice) & 3) << 24)
+#define   GEN8_MCR_SUBSLICE_MASK        GEN8_MCR_SUBSLICE(3)
 #define RING_IPEIR(base)        _MMIO((base)+0x64)
 #define RING_IPEHR(base)        _MMIO((base)+0x68)
 /*
@@ -2089,9 +2093,9 @@ enum skl_disp_power_wells {
 #define PM_VEBOX_CS_ERROR_INTERRUPT             (1 << 12) /* hsw+ */
 #define PM_VEBOX_USER_INTERRUPT                 (1 << 10) /* hsw+ */
 
-#define GT_PARITY_ERROR(dev) \
+#define GT_PARITY_ERROR(dev_priv) \
         (GT_RENDER_L3_PARITY_ERROR_INTERRUPT | \
-         (IS_HASWELL(dev) ? GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 : 0))
+         (IS_HASWELL(dev_priv) ? GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 : 0))
 
 /* These are all the "old" interrupts */
 #define ILK_BSD_USER_INTERRUPT                          (1<<5)
@@ -7327,6 +7331,10 @@ enum {
 #define   AUD_CONFIG_UPPER_N_MASK               (0xff << 20)
 #define   AUD_CONFIG_LOWER_N_SHIFT              4
 #define   AUD_CONFIG_LOWER_N_MASK               (0xfff << 4)
+#define   AUD_CONFIG_N_MASK                     (AUD_CONFIG_UPPER_N_MASK | AUD_CONFIG_LOWER_N_MASK)
+#define   AUD_CONFIG_N(n) \
+        (((((n) >> 12) & 0xff) << AUD_CONFIG_UPPER_N_SHIFT) |   \
+         (((n) & 0xfff) << AUD_CONFIG_LOWER_N_SHIFT))
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI_SHIFT     16
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK      (0xf << 16)
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI_25175     (0 << 16)
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index a0af170062b1..344cbf39cfa9 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -38,7 +38,7 @@ static void i915_save_display(struct drm_device *dev)
                 dev_priv->regfile.saveDSPARB = I915_READ(DSPARB);
 
         /* save FBC interval */
-        if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev))
+        if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv))
                 dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL);
 }
 
@@ -54,7 +54,7 @@ static void i915_restore_display(struct drm_device *dev)
         intel_fbc_global_disable(dev_priv);
 
         /* restore FBC interval */
-        if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev))
+        if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv))
                 I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL);
 
         i915_redisable_vga(dev);
@@ -70,7 +70,7 @@ int i915_save_state(struct drm_device *dev)
 
         i915_save_display(dev);
 
-        if (IS_GEN4(dev))
+        if (IS_GEN4(dev_priv))
                 pci_read_config_word(pdev, GCDGMBUS,
                                      &dev_priv->regfile.saveGCDGMBUS);
 
@@ -116,7 +116,7 @@ int i915_restore_state(struct drm_device *dev)
 
         i915_gem_restore_fences(dev);
 
-        if (IS_GEN4(dev))
+        if (IS_GEN4(dev_priv))
                 pci_write_config_word(pdev, GCDGMBUS,
                                       dev_priv->regfile.saveGCDGMBUS);
         i915_restore_display(dev);
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 1012eeea1324..47590ab08d7e 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -514,6 +514,8 @@ static const struct attribute *vlv_attrs[] = {
         NULL,
 };
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
                                 struct bin_attribute *attr, char *buf,
                                 loff_t off, size_t count)
@@ -571,6 +573,21 @@ static struct bin_attribute error_state_attr = {
         .write = error_state_write,
 };
 
+static void i915_setup_error_capture(struct device *kdev)
+{
+        if (sysfs_create_bin_file(&kdev->kobj, &error_state_attr))
+                DRM_ERROR("error_state sysfs setup failed\n");
+}
+
+static void i915_teardown_error_capture(struct device *kdev)
+{
+        sysfs_remove_bin_file(&kdev->kobj, &error_state_attr);
+}
+#else
+static void i915_setup_error_capture(struct device *kdev) {}
+static void i915_teardown_error_capture(struct device *kdev) {}
+#endif
+
 void i915_setup_sysfs(struct drm_i915_private *dev_priv)
 {
         struct device *kdev = dev_priv->drm.primary->kdev;
@@ -617,17 +634,15 @@ void i915_setup_sysfs(struct drm_i915_private *dev_priv)
         if (ret)
                 DRM_ERROR("RPS sysfs setup failed\n");
 
-        ret = sysfs_create_bin_file(&kdev->kobj,
-                                    &error_state_attr);
-        if (ret)
-                DRM_ERROR("error_state sysfs setup failed\n");
+        i915_setup_error_capture(kdev);
 }
 
 void i915_teardown_sysfs(struct drm_i915_private *dev_priv)
 {
         struct device *kdev = dev_priv->drm.primary->kdev;
 
-        sysfs_remove_bin_file(&kdev->kobj, &error_state_attr);
+        i915_teardown_error_capture(kdev);
+
         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 sysfs_remove_files(&kdev->kobj, vlv_attrs);
         else
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 6c70a5bfd7d8..7093cfbb62b1 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -81,7 +81,7 @@ static const struct {
         int clock;
         int n;
         int cts;
-} aud_ncts[] = {
+} hdmi_aud_ncts[] = {
         { 44100, TMDS_296M, 4459, 234375 },
         { 44100, TMDS_297M, 4704, 247500 },
         { 48000, TMDS_296M, 5824, 281250 },
@@ -121,45 +121,20 @@ static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted
         return hdmi_audio_clock[i].config;
 }
 
-static int audio_config_get_n(const struct drm_display_mode *mode, int rate)
+static int audio_config_hdmi_get_n(const struct drm_display_mode *adjusted_mode,
+                                   int rate)
 {
         int i;
 
-        for (i = 0; i < ARRAY_SIZE(aud_ncts); i++) {
-                if ((rate == aud_ncts[i].sample_rate) &&
-                        (mode->clock == aud_ncts[i].clock)) {
-                        return aud_ncts[i].n;
+        for (i = 0; i < ARRAY_SIZE(hdmi_aud_ncts); i++) {
+                if (rate == hdmi_aud_ncts[i].sample_rate &&
+                    adjusted_mode->crtc_clock == hdmi_aud_ncts[i].clock) {
+                        return hdmi_aud_ncts[i].n;
                 }
         }
         return 0;
 }
 
-static uint32_t audio_config_setup_n_reg(int n, uint32_t val)
-{
-        int n_low, n_up;
-        uint32_t tmp = val;
-
-        n_low = n & 0xfff;
-        n_up = (n >> 12) & 0xff;
-        tmp &= ~(AUD_CONFIG_UPPER_N_MASK | AUD_CONFIG_LOWER_N_MASK);
-        tmp |= ((n_up << AUD_CONFIG_UPPER_N_SHIFT) |
-                        (n_low << AUD_CONFIG_LOWER_N_SHIFT) |
-                        AUD_CONFIG_N_PROG_ENABLE);
-        return tmp;
-}
-
-/* check whether N/CTS/M need be set manually */
-static bool audio_rate_need_prog(struct intel_crtc *crtc,
-                                 const struct drm_display_mode *mode)
-{
-        if (((mode->clock == TMDS_297M) ||
-                 (mode->clock == TMDS_296M)) &&
-                intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI))
-                return true;
-        else
-                return false;
-}
-
 static bool intel_eld_uptodate(struct drm_connector *connector,
                                i915_reg_t reg_eldv, uint32_t bits_eldv,
                                i915_reg_t reg_elda, uint32_t bits_elda,
@@ -245,6 +220,65 @@ static void g4x_audio_codec_enable(struct drm_connector *connector,
         I915_WRITE(G4X_AUD_CNTL_ST, tmp);
 }
 
+static void
+hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
+                           const struct drm_display_mode *adjusted_mode)
+{
+        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+        enum pipe pipe = intel_crtc->pipe;
+        u32 tmp;
+
+        tmp = I915_READ(HSW_AUD_CFG(pipe));
+        tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
+        tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
+        tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
+        tmp |= AUD_CONFIG_N_VALUE_INDEX;
+
+        I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+}
+
+static void
+hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
+                             const struct drm_display_mode *adjusted_mode)
+{
+        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+        struct i915_audio_component *acomp = dev_priv->audio_component;
+        int rate = acomp ? acomp->aud_sample_rate[port] : 0;
+        enum pipe pipe = intel_crtc->pipe;
+        int n;
+        u32 tmp;
+
+        tmp = I915_READ(HSW_AUD_CFG(pipe));
+        tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
+        tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
+        tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
+        tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
+
+        if (adjusted_mode->crtc_clock == TMDS_296M ||
+            adjusted_mode->crtc_clock == TMDS_297M) {
+                n = audio_config_hdmi_get_n(adjusted_mode, rate);
+                if (n != 0) {
+                        tmp &= ~AUD_CONFIG_N_MASK;
+                        tmp |= AUD_CONFIG_N(n);
+                        tmp |= AUD_CONFIG_N_PROG_ENABLE;
+                } else {
+                        DRM_DEBUG_KMS("no suitable N value is found\n");
+                }
+        }
+
+        I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+}
+
+static void
+hsw_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
+                        const struct drm_display_mode *adjusted_mode)
+{
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
+                hsw_dp_audio_config_update(intel_crtc, port, adjusted_mode);
+        else
+                hsw_hdmi_audio_config_update(intel_crtc, port, adjusted_mode);
+}
+
 static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -276,20 +310,16 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 }
 
 static void hsw_audio_codec_enable(struct drm_connector *connector,
-                                   struct intel_encoder *encoder,
+                                   struct intel_encoder *intel_encoder,
                                    const struct drm_display_mode *adjusted_mode)
 {
         struct drm_i915_private *dev_priv = to_i915(connector->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
         enum pipe pipe = intel_crtc->pipe;
-        struct i915_audio_component *acomp = dev_priv->audio_component;
+        enum port port = intel_encoder->port;
         const uint8_t *eld = connector->eld;
-        struct intel_digital_port *intel_dig_port =
-                enc_to_dig_port(&encoder->base);
-        enum port port = intel_dig_port->port;
         uint32_t tmp;
         int len, i;
-        int n, rate;
 
         DRM_DEBUG_KMS("Enable audio codec on pipe %c, %u bytes ELD\n",
                       pipe_name(pipe), drm_eld_size(eld));
@@ -325,42 +355,17 @@ static void hsw_audio_codec_enable(struct drm_connector *connector,
         I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
 
         /* Enable timestamps */
-        tmp = I915_READ(HSW_AUD_CFG(pipe));
-        tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
-        tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-        if (intel_crtc_has_dp_encoder(intel_crtc->config))
-                tmp |= AUD_CONFIG_N_VALUE_INDEX;
-        else
-                tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
-
-        tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-        if (audio_rate_need_prog(intel_crtc, adjusted_mode)) {
-                if (!acomp)
-                        rate = 0;
-                else if (port >= PORT_A && port <= PORT_E)
-                        rate = acomp->aud_sample_rate[port];
-                else {
-                        DRM_ERROR("invalid port: %d\n", port);
-                        rate = 0;
-                }
-                n = audio_config_get_n(adjusted_mode, rate);
-                if (n != 0)
-                        tmp = audio_config_setup_n_reg(n, tmp);
-                else
-                        DRM_DEBUG_KMS("no suitable N value is found\n");
-        }
-
-        I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+        hsw_audio_config_update(intel_crtc, port, adjusted_mode);
 
         mutex_unlock(&dev_priv->av_mutex);
 }
 
-static void ilk_audio_codec_disable(struct intel_encoder *encoder)
+static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
 {
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-        enum port port = enc_to_dig_port(&encoder->base)->port;
+        struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
+        struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
         enum pipe pipe = intel_crtc->pipe;
+        enum port port = intel_encoder->port;
         uint32_t tmp, eldv;
         i915_reg_t aud_config, aud_cntrl_st2;
 
@@ -400,13 +405,13 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder)
 }
 
 static void ilk_audio_codec_enable(struct drm_connector *connector,
-                                   struct intel_encoder *encoder,
+                                   struct intel_encoder *intel_encoder,
                                    const struct drm_display_mode *adjusted_mode)
 {
         struct drm_i915_private *dev_priv = to_i915(connector->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-        enum port port = enc_to_dig_port(&encoder->base)->port;
+        struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
         enum pipe pipe = intel_crtc->pipe;
+        enum port port = intel_encoder->port;
         uint8_t *eld = connector->eld;
         uint32_t tmp, eldv;
         int len, i;
@@ -425,13 +430,13 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
          * infrastructure is not there yet.
          */
 
-        if (HAS_PCH_IBX(connector->dev)) {
+        if (HAS_PCH_IBX(dev_priv)) {
                 hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID(pipe);
                 aud_config = IBX_AUD_CFG(pipe);
                 aud_cntl_st = IBX_AUD_CNTL_ST(pipe);
                 aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
-        } else if (IS_VALLEYVIEW(connector->dev) ||
-                   IS_CHERRYVIEW(connector->dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv) ||
+                   IS_CHERRYVIEW(dev_priv)) {
                 hdmiw_hdmiedid = VLV_HDMIW_HDMIEDID(pipe);
                 aud_config = VLV_AUD_CFG(pipe);
                 aud_cntl_st = VLV_AUD_CNTL_ST(pipe);
@@ -490,11 +495,10 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
         struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
         struct drm_connector *connector;
-        struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
-        struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_encoder->port;
+        enum pipe pipe = crtc->pipe;
 
         connector = drm_select_eld(encoder);
         if (!connector)
@@ -518,13 +522,19 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
                                                      adjusted_mode);
 
         mutex_lock(&dev_priv->av_mutex);
-        intel_dig_port->audio_connector = connector;
+        intel_encoder->audio_connector = connector;
+
         /* referred in audio callbacks */
-        dev_priv->dig_port_map[port] = intel_encoder;
+        dev_priv->av_enc_map[pipe] = intel_encoder;
         mutex_unlock(&dev_priv->av_mutex);
 
+        /* audio drivers expect pipe = -1 to indicate Non-MST cases */
+        if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+                pipe = -1;
+
         if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
-                acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port);
+                acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+                                                 (int) port, (int) pipe);
 }
 
 /**
@@ -537,22 +547,27 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
 void intel_audio_codec_disable(struct intel_encoder *intel_encoder)
 {
         struct drm_encoder *encoder = &intel_encoder->base;
-        struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
-        struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_encoder->port;
+        struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
+        enum pipe pipe = crtc->pipe;
 
         if (dev_priv->display.audio_codec_disable)
                 dev_priv->display.audio_codec_disable(intel_encoder);
 
         mutex_lock(&dev_priv->av_mutex);
-        intel_dig_port->audio_connector = NULL;
-        dev_priv->dig_port_map[port] = NULL;
+        intel_encoder->audio_connector = NULL;
+        dev_priv->av_enc_map[pipe] = NULL;
         mutex_unlock(&dev_priv->av_mutex);
 
+        /* audio drivers expect pipe = -1 to indicate Non-MST cases */
+        if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+                pipe = -1;
+
         if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
-                acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port);
+                acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+                                                 (int) port, (int) pipe);
 }
 
 /**
@@ -627,74 +642,67 @@ static int i915_audio_component_get_cdclk_freq(struct device *kdev)
         return dev_priv->cdclk_freq;
 }
 
-static int i915_audio_component_sync_audio_rate(struct device *kdev,
-                                                int port, int rate)
+static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv,
+                                               int port, int pipe)
+{
+
+        if (WARN_ON(pipe >= I915_MAX_PIPES))
+                return NULL;
+
+        /* MST */
+        if (pipe >= 0)
+                return dev_priv->av_enc_map[pipe];
+
+        /* Non-MST */
+        for_each_pipe(dev_priv, pipe) {
+                struct intel_encoder *encoder;
+
+                encoder = dev_priv->av_enc_map[pipe];
+                if (encoder == NULL)
+                        continue;
+
+                if (port == encoder->port)
+                        return encoder;
+        }
+
+        return NULL;
+}
+
+static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
+                                                int pipe, int rate)
 {
         struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
         struct intel_encoder *intel_encoder;
         struct intel_crtc *crtc;
-        struct drm_display_mode *mode;
+        struct drm_display_mode *adjusted_mode;
         struct i915_audio_component *acomp = dev_priv->audio_component;
-        enum pipe pipe = INVALID_PIPE;
-        u32 tmp;
-        int n;
         int err = 0;
 
-        /* HSW, BDW, SKL, KBL need this fix */
-        if (!IS_SKYLAKE(dev_priv) &&
-            !IS_KABYLAKE(dev_priv) &&
-            !IS_BROADWELL(dev_priv) &&
-            !IS_HASWELL(dev_priv))
+        if (!HAS_DDI(dev_priv))
                 return 0;
 
         i915_audio_component_get_power(kdev);
         mutex_lock(&dev_priv->av_mutex);
+
         /* 1. get the pipe */
-        intel_encoder = dev_priv->dig_port_map[port];
-        /* intel_encoder might be NULL for DP MST */
+        intel_encoder = get_saved_enc(dev_priv, port, pipe);
         if (!intel_encoder || !intel_encoder->base.crtc ||
             intel_encoder->type != INTEL_OUTPUT_HDMI) {
-                DRM_DEBUG_KMS("no valid port %c\n", port_name(port));
+                DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port));
                 err = -ENODEV;
                 goto unlock;
         }
+
+        /* pipe passed from the audio driver will be -1 for Non-MST case */
         crtc = to_intel_crtc(intel_encoder->base.crtc);
         pipe = crtc->pipe;
-        if (pipe == INVALID_PIPE) {
-                DRM_DEBUG_KMS("no pipe for the port %c\n", port_name(port));
-                err = -ENODEV;
-                goto unlock;
-        }
 
-        DRM_DEBUG_KMS("pipe %c connects port %c\n",
-                                  pipe_name(pipe), port_name(port));
-        mode = &crtc->config->base.adjusted_mode;
+        adjusted_mode = &crtc->config->base.adjusted_mode;
 
         /* port must be valid now, otherwise the pipe will be invalid */
         acomp->aud_sample_rate[port] = rate;
 
-        /* 2. check whether to set the N/CTS/M manually or not */
-        if (!audio_rate_need_prog(crtc, mode)) {
-                tmp = I915_READ(HSW_AUD_CFG(pipe));
-                tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-                I915_WRITE(HSW_AUD_CFG(pipe), tmp);
-                goto unlock;
-        }
-
-        n = audio_config_get_n(mode, rate);
-        if (n == 0) {
-                DRM_DEBUG_KMS("Using automatic mode for N value on port %c\n",
-                                          port_name(port));
-                tmp = I915_READ(HSW_AUD_CFG(pipe));
-                tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-                I915_WRITE(HSW_AUD_CFG(pipe), tmp);
-                goto unlock;
-        }
-
-        /* 3. set the N/CTS/M */
-        tmp = I915_READ(HSW_AUD_CFG(pipe));
-        tmp = audio_config_setup_n_reg(n, tmp);
-        I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+        hsw_audio_config_update(crtc, port, adjusted_mode);
 
  unlock:
         mutex_unlock(&dev_priv->av_mutex);
@@ -703,27 +711,29 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev,
 }
 
 static int i915_audio_component_get_eld(struct device *kdev, int port,
-                                        bool *enabled,
+                                        int pipe, bool *enabled,
                                         unsigned char *buf, int max_bytes)
 {
         struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
         struct intel_encoder *intel_encoder;
-        struct intel_digital_port *intel_dig_port;
         const u8 *eld;
         int ret = -EINVAL;
 
         mutex_lock(&dev_priv->av_mutex);
-        intel_encoder = dev_priv->dig_port_map[port];
-        /* intel_encoder might be NULL for DP MST */
-        if (intel_encoder) {
-                ret = 0;
-                intel_dig_port = enc_to_dig_port(&intel_encoder->base);
-                *enabled = intel_dig_port->audio_connector != NULL;
-                if (*enabled) {
-                        eld = intel_dig_port->audio_connector->eld;
-                        ret = drm_eld_size(eld);
-                        memcpy(buf, eld, min(max_bytes, ret));
-                }
+
+        intel_encoder = get_saved_enc(dev_priv, port, pipe);
+        if (!intel_encoder) {
+                DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port));
+                mutex_unlock(&dev_priv->av_mutex);
+                return ret;
+        }
+
+        ret = 0;
+        *enabled = intel_encoder->audio_connector != NULL;
+        if (*enabled) {
+                eld = intel_encoder->audio_connector->eld;
+                ret = drm_eld_size(eld);
+                memcpy(buf, eld, min(max_bytes, ret));
         }
 
         mutex_unlock(&dev_priv->av_mutex);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index c6e69e4cfa83..5ab646ef8c9f 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -996,6 +996,10 @@ parse_mipi_sequence(struct drm_i915_private *dev_priv,
                         goto err;
                 }
 
+                /* Log about presence of sequences we won't run. */
+                if (seq_id == MIPI_SEQ_TEAR_ON || seq_id == MIPI_SEQ_TEAR_OFF)
+                        DRM_DEBUG_KMS("Unsupported sequence %u\n", seq_id);
+
                 dev_priv->vbt.dsi.sequence[seq_id] = data + index;
 
                 if (sequence->version >= 3)
@@ -1031,6 +1035,77 @@ static u8 translate_iboost(u8 val)
         return mapping[val];
 }
 
+static void sanitize_ddc_pin(struct drm_i915_private *dev_priv,
+                             enum port port)
+{
+        const struct ddi_vbt_port_info *info =
+                &dev_priv->vbt.ddi_port_info[port];
+        enum port p;
+
+        if (!info->alternate_ddc_pin)
+                return;
+
+        for_each_port_masked(p, (1 << port) - 1) {
+                struct ddi_vbt_port_info *i = &dev_priv->vbt.ddi_port_info[p];
+
+                if (info->alternate_ddc_pin != i->alternate_ddc_pin)
+                        continue;
+
+                DRM_DEBUG_KMS("port %c trying to use the same DDC pin (0x%x) as port %c, "
+                              "disabling port %c DVI/HDMI support\n",
+                              port_name(p), i->alternate_ddc_pin,
+                              port_name(port), port_name(p));
+
+                /*
+                 * If we have multiple ports supposedly sharing the
+                 * pin, then dvi/hdmi couldn't exist on the shared
+                 * port. Otherwise they share the same ddc bin and
+                 * system couldn't communicate with them separately.
+                 *
+                 * Due to parsing the ports in alphabetical order,
+                 * a higher port will always clobber a lower one.
+                 */
+                i->supports_dvi = false;
+                i->supports_hdmi = false;
+                i->alternate_ddc_pin = 0;
+        }
+}
+
+static void sanitize_aux_ch(struct drm_i915_private *dev_priv,
+                            enum port port)
+{
+        const struct ddi_vbt_port_info *info =
+                &dev_priv->vbt.ddi_port_info[port];
+        enum port p;
+
+        if (!info->alternate_aux_channel)
+                return;
+
+        for_each_port_masked(p, (1 << port) - 1) {
+                struct ddi_vbt_port_info *i = &dev_priv->vbt.ddi_port_info[p];
+
+                if (info->alternate_aux_channel != i->alternate_aux_channel)
+                        continue;
+
+                DRM_DEBUG_KMS("port %c trying to use the same AUX CH (0x%x) as port %c, "
+                              "disabling port %c DP support\n",
+                              port_name(p), i->alternate_aux_channel,
+                              port_name(port), port_name(p));
+
+                /*
+                 * If we have multiple ports supposedlt sharing the
+                 * aux channel, then DP couldn't exist on the shared
+                 * port. Otherwise they share the same aux channel
+                 * and system couldn't communicate with them separately.
+                 *
+                 * Due to parsing the ports in alphabetical order,
+                 * a higher port will always clobber a lower one.
+                 */
+                i->supports_dp = false;
+                i->alternate_aux_channel = 0;
+        }
+}
+
 static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
                            const struct bdb_header *bdb)
 {
@@ -1105,54 +1180,15 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
                 DRM_DEBUG_KMS("Port %c is internal DP\n", port_name(port));
 
         if (is_dvi) {
-                if (port == PORT_E) {
-                        info->alternate_ddc_pin = ddc_pin;
-                        /* if DDIE share ddc pin with other port, then
-                         * dvi/hdmi couldn't exist on the shared port.
-                         * Otherwise they share the same ddc bin and system
-                         * couldn't communicate with them seperately. */
-                        if (ddc_pin == DDC_PIN_B) {
-                                dev_priv->vbt.ddi_port_info[PORT_B].supports_dvi = 0;
-                                dev_priv->vbt.ddi_port_info[PORT_B].supports_hdmi = 0;
-                        } else if (ddc_pin == DDC_PIN_C) {
-                                dev_priv->vbt.ddi_port_info[PORT_C].supports_dvi = 0;
-                                dev_priv->vbt.ddi_port_info[PORT_C].supports_hdmi = 0;
-                        } else if (ddc_pin == DDC_PIN_D) {
-                                dev_priv->vbt.ddi_port_info[PORT_D].supports_dvi = 0;
-                                dev_priv->vbt.ddi_port_info[PORT_D].supports_hdmi = 0;
-                        }
-                } else if (ddc_pin == DDC_PIN_B && port != PORT_B)
-                        DRM_DEBUG_KMS("Unexpected DDC pin for port B\n");
-                else if (ddc_pin == DDC_PIN_C && port != PORT_C)
-                        DRM_DEBUG_KMS("Unexpected DDC pin for port C\n");
-                else if (ddc_pin == DDC_PIN_D && port != PORT_D)
-                        DRM_DEBUG_KMS("Unexpected DDC pin for port D\n");
+                info->alternate_ddc_pin = ddc_pin;
+
+                sanitize_ddc_pin(dev_priv, port);
         }
 
         if (is_dp) {
-                if (port == PORT_E) {
-                        info->alternate_aux_channel = aux_channel;
-                        /* if DDIE share aux channel with other port, then
-                         * DP couldn't exist on the shared port. Otherwise
-                         * they share the same aux channel and system
-                         * couldn't communicate with them seperately. */
-                        if (aux_channel == DP_AUX_A)
-                                dev_priv->vbt.ddi_port_info[PORT_A].supports_dp = 0;
-                        else if (aux_channel == DP_AUX_B)
-                                dev_priv->vbt.ddi_port_info[PORT_B].supports_dp = 0;
-                        else if (aux_channel == DP_AUX_C)
-                                dev_priv->vbt.ddi_port_info[PORT_C].supports_dp = 0;
-                        else if (aux_channel == DP_AUX_D)
-                                dev_priv->vbt.ddi_port_info[PORT_D].supports_dp = 0;
-                }
-                else if (aux_channel == DP_AUX_A && port != PORT_A)
-                        DRM_DEBUG_KMS("Unexpected AUX channel for port A\n");
-                else if (aux_channel == DP_AUX_B && port != PORT_B)
-                        DRM_DEBUG_KMS("Unexpected AUX channel for port B\n");
-                else if (aux_channel == DP_AUX_C && port != PORT_C)
-                        DRM_DEBUG_KMS("Unexpected AUX channel for port C\n");
-                else if (aux_channel == DP_AUX_D && port != PORT_D)
-                        DRM_DEBUG_KMS("Unexpected AUX channel for port D\n");
+                info->alternate_aux_channel = aux_channel;
+
+                sanitize_aux_ch(dev_priv, port);
         }
 
         if (bdb->version >= 158) {
@@ -1759,3 +1795,52 @@ intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv,
 
         return false;
 }
+
+/**
+ * intel_bios_is_lspcon_present - if LSPCON is attached on %port
+ * @dev_priv:   i915 device instance
+ * @port:       port to check
+ *
+ * Return true if LSPCON is present on this port
+ */
+bool
+intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv,
+                                enum port port)
+{
+        int i;
+
+        if (!HAS_LSPCON(dev_priv))
+                return false;
+
+        for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
+                if (!dev_priv->vbt.child_dev[i].common.lspcon)
+                        continue;
+
+                switch (dev_priv->vbt.child_dev[i].common.dvo_port) {
+                case DVO_PORT_DPA:
+                case DVO_PORT_HDMIA:
+                        if (port == PORT_A)
+                                return true;
+                        break;
+                case DVO_PORT_DPB:
+                case DVO_PORT_HDMIB:
+                        if (port == PORT_B)
+                                return true;
+                        break;
+                case DVO_PORT_DPC:
+                case DVO_PORT_HDMIC:
+                        if (port == PORT_C)
+                                return true;
+                        break;
+                case DVO_PORT_DPD:
+                case DVO_PORT_HDMID:
+                        if (port == PORT_D)
+                                return true;
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        return false;
+}
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 9bad14d22c95..23fc1042fed4 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -578,6 +578,36 @@ int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
         return 0;
 }
 
+static void cancel_fake_irq(struct intel_engine_cs *engine)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+        del_timer_sync(&b->hangcheck);
+        del_timer_sync(&b->fake_irq);
+        clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
+}
+
+void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+        cancel_fake_irq(engine);
+        spin_lock(&b->lock);
+
+        __intel_breadcrumbs_disable_irq(b);
+        if (intel_engine_has_waiter(engine)) {
+                b->timeout = wait_timeout();
+                __intel_breadcrumbs_enable_irq(b);
+                if (READ_ONCE(b->irq_posted))
+                        wake_up_process(b->first_wait->tsk);
+        } else {
+                /* sanitize the IMR and unmask any auxiliary interrupts */
+                irq_disable(engine);
+        }
+
+        spin_unlock(&b->lock);
+}
+
 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 {
         struct intel_breadcrumbs *b = &engine->breadcrumbs;
@@ -585,13 +615,13 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
         if (!IS_ERR_OR_NULL(b->signaler))
                 kthread_stop(b->signaler);
 
-        del_timer_sync(&b->hangcheck);
-        del_timer_sync(&b->fake_irq);
+        cancel_fake_irq(engine);
 }
 
 unsigned int intel_kick_waiters(struct drm_i915_private *i915)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         unsigned int mask = 0;
 
         /* To avoid the task_struct disappearing beneath us as we wake up
@@ -599,7 +629,7 @@ unsigned int intel_kick_waiters(struct drm_i915_private *i915)
          * RCU lock, i.e. as we call wake_up_process() we must be holding the
          * rcu_read_lock().
          */
-        for_each_engine(engine, i915)
+        for_each_engine(engine, i915, id)
                 if (unlikely(intel_engine_wakeup(engine)))
                         mask |= intel_engine_flag(engine);
 
@@ -609,9 +639,10 @@ unsigned int intel_kick_waiters(struct drm_i915_private *i915)
 unsigned int intel_kick_signalers(struct drm_i915_private *i915)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         unsigned int mask = 0;
 
-        for_each_engine(engine, i915) {
+        for_each_engine(engine, i915, id) {
                 if (unlikely(READ_ONCE(engine->breadcrumbs.first_signal))) {
                         wake_up_process(engine->breadcrumbs.signaler);
                         mask |= intel_engine_flag(engine);
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index 95a72771eea6..445108855275 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -273,7 +273,7 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
         enum pipe pipe = intel_crtc->pipe;
         int i;
 
-        if (HAS_GMCH_DISPLAY(dev)) {
+        if (HAS_GMCH_DISPLAY(dev_priv)) {
                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
                         assert_dsi_pll_enabled(dev_priv);
                 else
@@ -288,7 +288,7 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
                                 (drm_color_lut_extract(lut[i].green, 8) << 8) |
                                 drm_color_lut_extract(lut[i].blue, 8);
 
-                        if (HAS_GMCH_DISPLAY(dev))
+                        if (HAS_GMCH_DISPLAY(dev_priv))
                                 I915_WRITE(PALETTE(pipe, i), word);
                         else
                                 I915_WRITE(LGC_PALETTE(pipe, i), word);
@@ -297,7 +297,7 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
                 for (i = 0; i < 256; i++) {
                         uint32_t word = (i << 16) | (i << 8) | i;
 
-                        if (HAS_GMCH_DISPLAY(dev))
+                        if (HAS_GMCH_DISPLAY(dev_priv))
                                 I915_WRITE(PALETTE(pipe, i), word);
                         else
                                 I915_WRITE(LGC_PALETTE(pipe, i), word);
@@ -326,7 +326,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
          * Workaround : Do not read or write the pipe palette/gamma data while
          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
          */
-        if (IS_HASWELL(dev) && intel_crtc_state->ips_enabled &&
+        if (IS_HASWELL(dev_priv) && intel_crtc_state->ips_enabled &&
             (intel_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)) {
                 hsw_disable_ips(intel_crtc);
                 reenable_ips = true;
@@ -534,14 +534,14 @@ void intel_color_init(struct drm_crtc *crtc)
 
         drm_mode_crtc_set_gamma_size(crtc, 256);
 
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(dev_priv)) {
                 dev_priv->display.load_csc_matrix = cherryview_load_csc_matrix;
                 dev_priv->display.load_luts = cherryview_load_luts;
-        } else if (IS_HASWELL(dev)) {
+        } else if (IS_HASWELL(dev_priv)) {
                 dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix;
                 dev_priv->display.load_luts = haswell_load_luts;
-        } else if (IS_BROADWELL(dev) || IS_SKYLAKE(dev) ||
-                   IS_BROXTON(dev) || IS_KABYLAKE(dev)) {
+        } else if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv) ||
+                   IS_BROXTON(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix;
                 dev_priv->display.load_luts = broadwell_load_luts;
         } else {
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index dfbcf16b41df..a97151fcb9f4 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -84,7 +84,7 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder,
         if (!(tmp & ADPA_DAC_ENABLE))
                 goto out;
 
-        if (HAS_PCH_CPT(dev))
+        if (HAS_PCH_CPT(dev_priv))
                 *pipe = PORT_TO_PIPE_CPT(tmp);
         else
                 *pipe = PORT_TO_PIPE(tmp);
@@ -165,16 +165,16 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder,
                 adpa |= ADPA_VSYNC_ACTIVE_HIGH;
 
         /* For CPT allow 3 pipe config, for others just use A or B */
-        if (HAS_PCH_LPT(dev))
+        if (HAS_PCH_LPT(dev_priv))
                 ; /* Those bits don't exist here */
-        else if (HAS_PCH_CPT(dev))
+        else if (HAS_PCH_CPT(dev_priv))
                 adpa |= PORT_TRANS_SEL_CPT(crtc->pipe);
         else if (crtc->pipe == 0)
                 adpa |= ADPA_PIPE_A_SELECT;
         else
                 adpa |= ADPA_PIPE_B_SELECT;
 
-        if (!HAS_PCH_SPLIT(dev))
+        if (!HAS_PCH_SPLIT(dev_priv))
                 I915_WRITE(BCLRPAT(crtc->pipe), 0);
 
         switch (mode) {
@@ -241,7 +241,8 @@ intel_crt_mode_valid(struct drm_connector *connector,
                      struct drm_display_mode *mode)
 {
         struct drm_device *dev = connector->dev;
-        int max_dotclk = to_i915(dev)->max_dotclk_freq;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        int max_dotclk = dev_priv->max_dotclk_freq;
         int max_clock;
 
         if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
@@ -250,15 +251,15 @@ intel_crt_mode_valid(struct drm_connector *connector,
         if (mode->clock < 25000)
                 return MODE_CLOCK_LOW;
 
-        if (HAS_PCH_LPT(dev))
+        if (HAS_PCH_LPT(dev_priv))
                 max_clock = 180000;
-        else if (IS_VALLEYVIEW(dev))
+        else if (IS_VALLEYVIEW(dev_priv))
                 /*
                  * 270 MHz due to current DPLL limits,
                  * DAC limit supposedly 355 MHz.
                  */
                 max_clock = 270000;
-        else if (IS_GEN3(dev) || IS_GEN4(dev))
+        else if (IS_GEN3(dev_priv) || IS_GEN4(dev_priv))
                 max_clock = 400000;
         else
                 max_clock = 350000;
@@ -269,7 +270,7 @@ intel_crt_mode_valid(struct drm_connector *connector,
                 return MODE_CLOCK_HIGH;
 
         /* The FDI receiver on LPT only supports 8bpc and only has 2 lanes. */
-        if (HAS_PCH_LPT(dev) &&
+        if (HAS_PCH_LPT(dev_priv) &&
             (ironlake_get_lanes_required(mode->clock, 270000, 24) > 2))
                 return MODE_CLOCK_HIGH;
 
@@ -280,13 +281,13 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder,
                                      struct intel_crtc_state *pipe_config,
                                      struct drm_connector_state *conn_state)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 pipe_config->has_pch_encoder = true;
 
         /* LPT FDI RX only supports 8bpc. */
-        if (HAS_PCH_LPT(dev)) {
+        if (HAS_PCH_LPT(dev_priv)) {
                 if (pipe_config->bw_constrained && pipe_config->pipe_bpp < 24) {
                         DRM_DEBUG_KMS("LPT only supports 24bpp\n");
                         return false;
@@ -296,7 +297,7 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder,
         }
 
         /* FDI must always be 2.7 GHz */
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev_priv))
                 pipe_config->port_clock = 135000 * 2;
 
         return true;
@@ -312,7 +313,7 @@ static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector)
 
         /* The first time through, trigger an explicit detection cycle */
         if (crt->force_hotplug_required) {
-                bool turn_off_dac = HAS_PCH_SPLIT(dev);
+                bool turn_off_dac = HAS_PCH_SPLIT(dev_priv);
                 u32 save_adpa;
 
                 crt->force_hotplug_required = 0;
@@ -419,10 +420,10 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
         bool ret = false;
         int i, tries = 0;
 
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 return intel_ironlake_crt_detect_hotplug(connector);
 
-        if (IS_VALLEYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv))
                 return valleyview_crt_detect_hotplug(connector);
 
         /*
@@ -430,7 +431,7 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
          * to get a reliable result.
          */
 
-        if (IS_G4X(dev) && !IS_GM45(dev))
+        if (IS_G4X(dev_priv) && !IS_GM45(dev_priv))
                 tries = 2;
         else
                 tries = 1;
@@ -566,7 +567,7 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe)
         /* Set the border color to purple. */
         I915_WRITE(bclrpat_reg, 0x500050);
 
-        if (!IS_GEN2(dev)) {
+        if (!IS_GEN2(dev_priv)) {
                 uint32_t pipeconf = I915_READ(pipeconf_reg);
                 I915_WRITE(pipeconf_reg, pipeconf | PIPECONF_FORCE_BORDER);
                 POSTING_READ(pipeconf_reg);
@@ -643,6 +644,32 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe)
         return status;
 }
 
+static int intel_spurious_crt_detect_dmi_callback(const struct dmi_system_id *id)
+{
+        DRM_DEBUG_DRIVER("Skipping CRT detection for %s\n", id->ident);
+        return 1;
+}
+
+static const struct dmi_system_id intel_spurious_crt_detect[] = {
+        {
+                .callback = intel_spurious_crt_detect_dmi_callback,
+                .ident = "ACER ZGB",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "ACER"),
+                        DMI_MATCH(DMI_PRODUCT_NAME, "ZGB"),
+                },
+        },
+        {
+                .callback = intel_spurious_crt_detect_dmi_callback,
+                .ident = "Intel DZ77BH-55K",
+                .matches = {
+                        DMI_MATCH(DMI_BOARD_VENDOR, "Intel Corporation"),
+                        DMI_MATCH(DMI_BOARD_NAME, "DZ77BH-55K"),
+                },
+        },
+        { }
+};
+
 static enum drm_connector_status
 intel_crt_detect(struct drm_connector *connector, bool force)
 {
@@ -659,6 +686,10 @@ intel_crt_detect(struct drm_connector *connector, bool force)
                       connector->base.id, connector->name,
                       force);
 
+        /* Skip machines without VGA that falsely report hotplug events */
+        if (dmi_check_system(intel_spurious_crt_detect))
+                return connector_status_disconnected;
+
         power_domain = intel_display_port_power_domain(intel_encoder);
         intel_display_power_get(dev_priv, power_domain);
 
@@ -740,7 +771,7 @@ static int intel_crt_get_modes(struct drm_connector *connector)
 
         i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin);
         ret = intel_crt_ddc_get_modes(connector, i2c);
-        if (ret || !IS_G4X(dev))
+        if (ret || !IS_G4X(dev_priv))
                 goto out;
 
         /* Try to probe digital port for output in DVI-I -> VGA mode. */
@@ -808,32 +839,6 @@ static const struct drm_encoder_funcs intel_crt_enc_funcs = {
         .destroy = intel_encoder_destroy,
 };
 
-static int intel_no_crt_dmi_callback(const struct dmi_system_id *id)
-{
-        DRM_INFO("Skipping CRT initialization for %s\n", id->ident);
-        return 1;
-}
-
-static const struct dmi_system_id intel_no_crt[] = {
-        {
-                .callback = intel_no_crt_dmi_callback,
-                .ident = "ACER ZGB",
-                .matches = {
-                        DMI_MATCH(DMI_SYS_VENDOR, "ACER"),
-                        DMI_MATCH(DMI_PRODUCT_NAME, "ZGB"),
-                },
-        },
-        {
-                .callback = intel_no_crt_dmi_callback,
-                .ident = "DELL XPS 8700",
-                .matches = {
-                        DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-                        DMI_MATCH(DMI_PRODUCT_NAME, "XPS 8700"),
-                },
-        },
-        { }
-};
-
 void intel_crt_init(struct drm_device *dev)
 {
         struct drm_connector *connector;
@@ -843,13 +848,9 @@ void intel_crt_init(struct drm_device *dev)
         i915_reg_t adpa_reg;
         u32 adpa;
 
-        /* Skip machines without VGA that falsely report hotplug events */
-        if (dmi_check_system(intel_no_crt))
-                return;
-
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 adpa_reg = PCH_ADPA;
-        else if (IS_VALLEYVIEW(dev))
+        else if (IS_VALLEYVIEW(dev_priv))
                 adpa_reg = VLV_ADPA;
         else
                 adpa_reg = ADPA;
@@ -893,12 +894,12 @@ void intel_crt_init(struct drm_device *dev)
 
         crt->base.type = INTEL_OUTPUT_ANALOG;
         crt->base.cloneable = (1 << INTEL_OUTPUT_DVO) | (1 << INTEL_OUTPUT_HDMI);
-        if (IS_I830(dev))
+        if (IS_I830(dev_priv))
                 crt->base.crtc_mask = (1 << 0);
         else
                 crt->base.crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 connector->interlace_allowed = 0;
         else
                 connector->interlace_allowed = 1;
@@ -907,20 +908,23 @@ void intel_crt_init(struct drm_device *dev)
         crt->adpa_reg = adpa_reg;
 
         crt->base.compute_config = intel_crt_compute_config;
-        if (HAS_PCH_SPLIT(dev)) {
+        if (HAS_PCH_SPLIT(dev_priv)) {
                 crt->base.disable = pch_disable_crt;
                 crt->base.post_disable = pch_post_disable_crt;
         } else {
                 crt->base.disable = intel_disable_crt;
         }
         crt->base.enable = intel_enable_crt;
-        if (I915_HAS_HOTPLUG(dev))
+        if (I915_HAS_HOTPLUG(dev) &&
+            !dmi_check_system(intel_spurious_crt_detect))
                 crt->base.hpd_pin = HPD_CRT;
-        if (HAS_DDI(dev)) {
+        if (HAS_DDI(dev_priv)) {
+                crt->base.port = PORT_E;
                 crt->base.get_config = hsw_crt_get_config;
                 crt->base.get_hw_state = intel_ddi_get_hw_state;
                 crt->base.post_disable = hsw_post_disable_crt;
         } else {
+                crt->base.port = PORT_NONE;
                 crt->base.get_config = intel_crt_get_config;
                 crt->base.get_hw_state = intel_crt_get_hw_state;
         }
@@ -941,7 +945,7 @@ void intel_crt_init(struct drm_device *dev)
          * polarity and link reversal bits or not, instead of relying on the
          * BIOS.
          */
-        if (HAS_PCH_LPT(dev)) {
+        if (HAS_PCH_LPT(dev_priv)) {
                 u32 fdi_config = FDI_RX_POLARITY_REVERSED_LPT |
                                  FDI_RX_LINK_REVERSAL_OVERRIDE;
 
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 15d47c87def6..fb18d699ce10 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -167,8 +167,47 @@ static const struct ddi_buf_trans skl_y_ddi_translations_dp[] = {
         { 0x80005012, 0x000000C0, 0x3 },
 };
 
+/* Kabylake H and S */
+static const struct ddi_buf_trans kbl_ddi_translations_dp[] = {
+        { 0x00002016, 0x000000A0, 0x0 },
+        { 0x00005012, 0x0000009B, 0x0 },
+        { 0x00007011, 0x00000088, 0x0 },
+        { 0x80009010, 0x000000C0, 0x1 },
+        { 0x00002016, 0x0000009B, 0x0 },
+        { 0x00005012, 0x00000088, 0x0 },
+        { 0x80007011, 0x000000C0, 0x1 },
+        { 0x00002016, 0x00000097, 0x0 },
+        { 0x80005012, 0x000000C0, 0x1 },
+};
+
+/* Kabylake U */
+static const struct ddi_buf_trans kbl_u_ddi_translations_dp[] = {
+        { 0x0000201B, 0x000000A1, 0x0 },
+        { 0x00005012, 0x00000088, 0x0 },
+        { 0x80007011, 0x000000CD, 0x3 },
+        { 0x80009010, 0x000000C0, 0x3 },
+        { 0x0000201B, 0x0000009D, 0x0 },
+        { 0x80005012, 0x000000C0, 0x3 },
+        { 0x80007011, 0x000000C0, 0x3 },
+        { 0x00002016, 0x0000004F, 0x0 },
+        { 0x80005012, 0x000000C0, 0x3 },
+};
+
+/* Kabylake Y */
+static const struct ddi_buf_trans kbl_y_ddi_translations_dp[] = {
+        { 0x00001017, 0x000000A1, 0x0 },
+        { 0x00005012, 0x00000088, 0x0 },
+        { 0x80007011, 0x000000CD, 0x3 },
+        { 0x8000800F, 0x000000C0, 0x3 },
+        { 0x00001017, 0x0000009D, 0x0 },
+        { 0x80005012, 0x000000C0, 0x3 },
+        { 0x80007011, 0x000000C0, 0x3 },
+        { 0x00001017, 0x0000004C, 0x0 },
+        { 0x80005012, 0x000000C0, 0x3 },
+};
+
 /*
- * Skylake H and S
+ * Skylake/Kabylake H and S
  * eDP 1.4 low vswing translation parameters
  */
 static const struct ddi_buf_trans skl_ddi_translations_edp[] = {
@@ -185,7 +224,7 @@ static const struct ddi_buf_trans skl_ddi_translations_edp[] = {
 };
 
 /*
- * Skylake U
+ * Skylake/Kabylake U
  * eDP 1.4 low vswing translation parameters
  */
 static const struct ddi_buf_trans skl_u_ddi_translations_edp[] = {
@@ -202,7 +241,7 @@ static const struct ddi_buf_trans skl_u_ddi_translations_edp[] = {
 };
 
 /*
- * Skylake Y
+ * Skylake/Kabylake Y
  * eDP 1.4 low vswing translation parameters
  */
 static const struct ddi_buf_trans skl_y_ddi_translations_edp[] = {
@@ -218,7 +257,7 @@ static const struct ddi_buf_trans skl_y_ddi_translations_edp[] = {
         { 0x00000018, 0x0000008A, 0x0 },
 };
 
-/* Skylake U, H and S */
+/* Skylake/Kabylake U, H and S */
 static const struct ddi_buf_trans skl_ddi_translations_hdmi[] = {
         { 0x00000018, 0x000000AC, 0x0 },
         { 0x00005012, 0x0000009D, 0x0 },
@@ -233,7 +272,7 @@ static const struct ddi_buf_trans skl_ddi_translations_hdmi[] = {
         { 0x80000018, 0x000000C0, 0x1 },
 };
 
-/* Skylake Y */
+/* Skylake/Kabylake Y */
 static const struct ddi_buf_trans skl_y_ddi_translations_hdmi[] = {
         { 0x00000018, 0x000000A1, 0x0 },
         { 0x00005012, 0x000000DF, 0x0 },
@@ -334,10 +373,10 @@ bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 static const struct ddi_buf_trans *
 skl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries)
 {
-        if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv)) {
+        if (IS_SKL_ULX(dev_priv)) {
                 *n_entries = ARRAY_SIZE(skl_y_ddi_translations_dp);
                 return skl_y_ddi_translations_dp;
-        } else if (IS_SKL_ULT(dev_priv) || IS_KBL_ULT(dev_priv)) {
+        } else if (IS_SKL_ULT(dev_priv)) {
                 *n_entries = ARRAY_SIZE(skl_u_ddi_translations_dp);
                 return skl_u_ddi_translations_dp;
         } else {
@@ -347,6 +386,21 @@ skl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries)
 }
 
 static const struct ddi_buf_trans *
+kbl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries)
+{
+        if (IS_KBL_ULX(dev_priv)) {
+                *n_entries = ARRAY_SIZE(kbl_y_ddi_translations_dp);
+                return kbl_y_ddi_translations_dp;
+        } else if (IS_KBL_ULT(dev_priv)) {
+                *n_entries = ARRAY_SIZE(kbl_u_ddi_translations_dp);
+                return kbl_u_ddi_translations_dp;
+        } else {
+                *n_entries = ARRAY_SIZE(kbl_ddi_translations_dp);
+                return kbl_ddi_translations_dp;
+        }
+}
+
+static const struct ddi_buf_trans *
 skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 {
         if (dev_priv->vbt.edp.low_vswing) {
@@ -362,7 +416,10 @@ skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
                 }
         }
 
-        return skl_get_buf_trans_dp(dev_priv, n_entries);
+        if (IS_KABYLAKE(dev_priv))
+                return kbl_get_buf_trans_dp(dev_priv, n_entries);
+        else
+                return skl_get_buf_trans_dp(dev_priv, n_entries);
 }
 
 static const struct ddi_buf_trans *
@@ -430,21 +487,18 @@ void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder)
         if (IS_BROXTON(dev_priv))
                 return;
 
-        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+        if (IS_KABYLAKE(dev_priv)) {
+                ddi_translations_fdi = NULL;
+                ddi_translations_dp =
+                                kbl_get_buf_trans_dp(dev_priv, &n_dp_entries);
+                ddi_translations_edp =
+                                skl_get_buf_trans_edp(dev_priv, &n_edp_entries);
+        } else if (IS_SKYLAKE(dev_priv)) {
                 ddi_translations_fdi = NULL;
                 ddi_translations_dp =
                                 skl_get_buf_trans_dp(dev_priv, &n_dp_entries);
                 ddi_translations_edp =
                                 skl_get_buf_trans_edp(dev_priv, &n_edp_entries);
-
-                /* If we're boosting the current, set bit 31 of trans1 */
-                if (dev_priv->vbt.ddi_port_info[port].dp_boost_level)
-                        iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
-
-                if (WARN_ON(encoder->type == INTEL_OUTPUT_EDP &&
-                            port != PORT_A && port != PORT_E &&
-                            n_edp_entries > 9))
-                        n_edp_entries = 9;
         } else if (IS_BROADWELL(dev_priv)) {
                 ddi_translations_fdi = bdw_ddi_translations_fdi;
                 ddi_translations_dp = bdw_ddi_translations_dp;
@@ -464,6 +518,17 @@ void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder)
                 n_dp_entries = ARRAY_SIZE(bdw_ddi_translations_dp);
         }
 
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+                /* If we're boosting the current, set bit 31 of trans1 */
+                if (dev_priv->vbt.ddi_port_info[port].dp_boost_level)
+                        iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
+
+                if (WARN_ON(encoder->type == INTEL_OUTPUT_EDP &&
+                            port != PORT_A && port != PORT_E &&
+                            n_edp_entries > 9))
+                        n_edp_entries = 9;
+        }
+
         switch (encoder->type) {
         case INTEL_OUTPUT_EDP:
                 ddi_translations = ddi_translations_edp;
@@ -1020,13 +1085,13 @@ static void bxt_ddi_clock_get(struct intel_encoder *encoder,
 void intel_ddi_clock_get(struct intel_encoder *encoder,
                          struct intel_crtc_state *pipe_config)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (INTEL_INFO(dev)->gen <= 8)
+        if (INTEL_GEN(dev_priv) <= 8)
                 hsw_ddi_clock_get(encoder, pipe_config);
-        else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+        else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
                 skl_ddi_clock_get(encoder, pipe_config);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 bxt_ddi_clock_get(encoder, pipe_config);
 }
 
@@ -1081,14 +1146,14 @@ bxt_ddi_pll_select(struct intel_crtc *intel_crtc,
 bool intel_ddi_pll_select(struct intel_crtc *intel_crtc,
                           struct intel_crtc_state *crtc_state)
 {
-        struct drm_device *dev = intel_crtc->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
         struct intel_encoder *intel_encoder =
                 intel_ddi_get_crtc_new_encoder(crtc_state);
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
                 return skl_ddi_pll_select(intel_crtc, crtc_state,
                                           intel_encoder);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 return bxt_ddi_pll_select(intel_crtc, crtc_state,
                                           intel_encoder);
         else
@@ -1189,7 +1254,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
                          * eDP when not using the panel fitter, and when not
                          * using motion blur mitigation (which we don't
                          * support). */
-                        if (IS_HASWELL(dev) &&
+                        if (IS_HASWELL(dev_priv) &&
                             (intel_crtc->config->pch_pfit.enabled ||
                              intel_crtc->config->pch_pfit.force_thru))
                                 temp |= TRANS_DDI_EDP_INPUT_A_ONOFF;
@@ -1434,7 +1499,12 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, u32 level)
                 if (dp_iboost) {
                         iboost = dp_iboost;
                 } else {
-                        ddi_translations = skl_get_buf_trans_dp(dev_priv, &n_entries);
+                        if (IS_KABYLAKE(dev_priv))
+                                ddi_translations = kbl_get_buf_trans_dp(dev_priv,
+                                                                        &n_entries);
+                        else
+                                ddi_translations = skl_get_buf_trans_dp(dev_priv,
+                                                                        &n_entries);
                         iboost = ddi_translations[level].i_boost;
                 }
         } else if (type == INTEL_OUTPUT_EDP) {
@@ -1742,7 +1812,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
                 intel_edp_panel_off(intel_dp);
         }
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
                 I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) |
                                         DPLL_CTRL2_DDI_CLK_OFF(port)));
         else if (INTEL_INFO(dev)->gen < 9)
@@ -2438,7 +2508,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
         struct intel_digital_port *intel_dig_port;
         struct intel_encoder *intel_encoder;
         struct drm_encoder *encoder;
-        bool init_hdmi, init_dp;
+        bool init_hdmi, init_dp, init_lspcon = false;
         int max_lanes;
 
         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) {
@@ -2470,6 +2540,19 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
         init_hdmi = (dev_priv->vbt.ddi_port_info[port].supports_dvi ||
                      dev_priv->vbt.ddi_port_info[port].supports_hdmi);
         init_dp = dev_priv->vbt.ddi_port_info[port].supports_dp;
+
+        if (intel_bios_is_lspcon_present(dev_priv, port)) {
+                /*
+                 * Lspcon device needs to be driven with DP connector
+                 * with special detection sequence. So make sure DP
+                 * is initialized before lspcon.
+                 */
+                init_dp = true;
+                init_lspcon = true;
+                init_hdmi = false;
+                DRM_DEBUG_KMS("VBT says port %c has lspcon\n", port_name(port));
+        }
+
         if (!init_dp && !init_hdmi) {
                 DRM_DEBUG_KMS("VBT says port %c is not DVI/HDMI/DP compatible, respect it\n",
                               port_name(port));
@@ -2509,7 +2592,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
          * configuration so that we use the proper lane count for our
          * calculations.
          */
-        if (IS_BROXTON(dev) && port == PORT_A) {
+        if (IS_BROXTON(dev_priv) && port == PORT_A) {
                 if (!(intel_dig_port->saved_port_bits & DDI_A_4_LANES)) {
                         DRM_DEBUG_KMS("BXT BIOS forgot to set DDI_A_4_LANES for port A; fixing\n");
                         intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
@@ -2520,6 +2603,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
         intel_dig_port->max_lanes = max_lanes;
 
         intel_encoder->type = INTEL_OUTPUT_UNKNOWN;
+        intel_encoder->port = port;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
         intel_encoder->cloneable = 0;
 
@@ -2532,7 +2616,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
                  * On BXT A0/A1, sw needs to activate DDIA HPD logic and
                  * interrupts to check the external panel connection.
                  */
-                if (IS_BXT_REVID(dev, 0, BXT_REVID_A1) && port == PORT_B)
+                if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) && port == PORT_B)
                         dev_priv->hotplug.irq_port[PORT_A] = intel_dig_port;
                 else
                         dev_priv->hotplug.irq_port[port] = intel_dig_port;
@@ -2545,6 +2629,20 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
                         goto err;
         }
 
+        if (init_lspcon) {
+                if (lspcon_init(intel_dig_port))
+                        /* TODO: handle hdmi info frame part */
+                        DRM_DEBUG_KMS("LSPCON init success on port %c\n",
+                                port_name(port));
+                else
+                        /*
+                         * LSPCON init faied, but DP init was success, so
+                         * lets try to drive as DP++ port.
+                         */
+                        DRM_ERROR("LSPCON init failed on port %c\n",
+                                port_name(port));
+        }
+
         return;
 
 err:
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 73b6858600ac..d6a8f11813d5 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -28,20 +28,14 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv)
 {
         const struct intel_device_info *info = &dev_priv->info;
 
-#define PRINT_S(name) "%s"
-#define SEP_EMPTY
-#define PRINT_FLAG(name) info->name ? #name "," : ""
-#define SEP_COMMA ,
-        DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags="
-                         DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
+        DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x",
                          info->gen,
                          dev_priv->drm.pdev->device,
-                         dev_priv->drm.pdev->revision,
-                         DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
-#undef PRINT_S
-#undef SEP_EMPTY
+                         dev_priv->drm.pdev->revision);
+#define PRINT_FLAG(name) \
+        DRM_DEBUG_DRIVER("i915 device info: " #name ": %s", yesno(info->name))
+        DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
-#undef SEP_COMMA
 }
 
 static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv)
@@ -192,7 +186,7 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
         struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
         const int s_max = 3, ss_max = 3, eu_max = 8;
         int s, ss;
-        u32 fuse2, eu_disable[s_max];
+        u32 fuse2, eu_disable[3]; /* s_max */
 
         fuse2 = I915_READ(GEN8_FUSE2);
         sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 6c5c36eba6cb..6f8f6ec5b27a 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -600,7 +600,7 @@ int chv_calc_dpll_params(int refclk, struct dpll *clock)
  * the given connectors.
  */
 
-static bool intel_PLL_is_valid(struct drm_device *dev,
+static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
                                const struct intel_limit *limit,
                                const struct dpll *clock)
 {
@@ -613,12 +613,13 @@ static bool intel_PLL_is_valid(struct drm_device *dev,
         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
                 INTELPllInvalid("m1 out of range\n");
 
-        if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) &&
-            !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev))
+        if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
+            !IS_CHERRYVIEW(dev_priv) && !IS_BROXTON(dev_priv))
                 if (clock->m1 <= clock->m2)
                         INTELPllInvalid("m1 <= m2\n");
 
-        if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev)) {
+        if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
+            !IS_BROXTON(dev_priv)) {
                 if (clock->p < limit->p.min || limit->p.max < clock->p)
                         INTELPllInvalid("p out of range\n");
                 if (clock->m < limit->m.min || limit->m.max < clock->m)
@@ -698,7 +699,8 @@ i9xx_find_best_dpll(const struct intel_limit *limit,
                                         int this_err;
 
                                         i9xx_calc_dpll_params(refclk, &clock);
-                                        if (!intel_PLL_is_valid(dev, limit,
+                                        if (!intel_PLL_is_valid(to_i915(dev),
+                                                                limit,
                                                                 &clock))
                                                 continue;
                                         if (match_clock &&
@@ -753,7 +755,8 @@ pnv_find_best_dpll(const struct intel_limit *limit,
                                         int this_err;
 
                                         pnv_calc_dpll_params(refclk, &clock);
-                                        if (!intel_PLL_is_valid(dev, limit,
+                                        if (!intel_PLL_is_valid(to_i915(dev),
+                                                                limit,
                                                                 &clock))
                                                 continue;
                                         if (match_clock &&
@@ -813,7 +816,8 @@ g4x_find_best_dpll(const struct intel_limit *limit,
                                         int this_err;
 
                                         i9xx_calc_dpll_params(refclk, &clock);
-                                        if (!intel_PLL_is_valid(dev, limit,
+                                        if (!intel_PLL_is_valid(to_i915(dev),
+                                                                limit,
                                                                 &clock))
                                                 continue;
 
@@ -845,7 +849,7 @@ static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
          * For CHV ignore the error and consider only the P value.
          * Prefer a bigger P value based on HW requirements.
          */
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(to_i915(dev))) {
                 *error_ppm = 0;
 
                 return calculated_clock->p > best_clock->p;
@@ -909,7 +913,8 @@ vlv_find_best_dpll(const struct intel_limit *limit,
 
                                         vlv_calc_dpll_params(refclk, &clock);
 
-                                        if (!intel_PLL_is_valid(dev, limit,
+                                        if (!intel_PLL_is_valid(to_i915(dev),
+                                                                limit,
                                                                 &clock))
                                                 continue;
 
@@ -977,7 +982,7 @@ chv_find_best_dpll(const struct intel_limit *limit,
 
                         chv_calc_dpll_params(refclk, &clock);
 
-                        if (!intel_PLL_is_valid(dev, limit, &clock))
+                        if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
                                 continue;
 
                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
@@ -1040,7 +1045,7 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
         u32 line1, line2;
         u32 line_mask;
 
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 line_mask = DSL_LINEMASK_GEN2;
         else
                 line_mask = DSL_LINEMASK_GEN3;
@@ -1187,19 +1192,17 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
                         onoff(state), onoff(cur_state));
 }
 
-void assert_panel_unlocked(struct drm_i915_private *dev_priv,
-                           enum pipe pipe)
+void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-        struct drm_device *dev = &dev_priv->drm;
         i915_reg_t pp_reg;
         u32 val;
         enum pipe panel_pipe = PIPE_A;
         bool locked = true;
 
-        if (WARN_ON(HAS_DDI(dev)))
+        if (WARN_ON(HAS_DDI(dev_priv)))
                 return;
 
-        if (HAS_PCH_SPLIT(dev)) {
+        if (HAS_PCH_SPLIT(dev_priv)) {
                 u32 port_sel;
 
                 pp_reg = PP_CONTROL(0);
@@ -1209,7 +1212,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
                     I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
                         panel_pipe = PIPE_B;
                 /* XXX: else fix for eDP */
-        } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 /* presumably write lock depends on pipe, not port select */
                 pp_reg = PP_CONTROL(pipe);
                 panel_pipe = pipe;
@@ -1232,10 +1235,9 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 static void assert_cursor(struct drm_i915_private *dev_priv,
                           enum pipe pipe, bool state)
 {
-        struct drm_device *dev = &dev_priv->drm;
         bool cur_state;
 
-        if (IS_845G(dev) || IS_I865G(dev))
+        if (IS_845G(dev_priv) || IS_I865G(dev_priv))
                 cur_state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
         else
                 cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
@@ -1330,7 +1332,7 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
                              "plane %d assertion failure, should be off on pipe %c but is still active\n",
                              sprite, pipe_name(pipe));
                 }
-        } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 for_each_sprite(dev_priv, pipe, sprite) {
                         u32 val = I915_READ(SPCNTR(pipe, sprite));
                         I915_STATE_WARN(val & SP_ENABLE,
@@ -1619,11 +1621,11 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
         assert_pipe_disabled(dev_priv, crtc->pipe);
 
         /* PLL is protected by panel, make sure we can write it */
-        if (IS_MOBILE(dev) && !IS_I830(dev))
+        if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
                 assert_panel_unlocked(dev_priv, crtc->pipe);
 
         /* Enable DVO 2x clock on both PLLs if necessary */
-        if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
+        if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev) > 0) {
                 /*
                  * It appears to be important that we don't enable this
                  * for the current pipe before otherwise configuring the
@@ -1688,7 +1690,7 @@ static void i9xx_disable_pll(struct intel_crtc *crtc)
         enum pipe pipe = crtc->pipe;
 
         /* Disable DVO 2x clock on both PLLs if necessary */
-        if (IS_I830(dev) &&
+        if (IS_I830(dev_priv) &&
             intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) &&
             !intel_num_dvo_pipes(dev)) {
                 I915_WRITE(DPLL(PIPE_B),
@@ -1786,7 +1788,6 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
                                            enum pipe pipe)
 {
-        struct drm_device *dev = &dev_priv->drm;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         i915_reg_t reg;
@@ -1799,7 +1800,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
         assert_fdi_tx_enabled(dev_priv, pipe);
         assert_fdi_rx_enabled(dev_priv, pipe);
 
-        if (HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_CPT(dev_priv)) {
                 /* Workaround: Set the timing override bit before enabling the
                  * pch transcoder. */
                 reg = TRANS_CHICKEN2(pipe);
@@ -1877,7 +1878,6 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
                                             enum pipe pipe)
 {
-        struct drm_device *dev = &dev_priv->drm;
         i915_reg_t reg;
         uint32_t val;
 
@@ -1898,7 +1898,7 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
                                     50))
                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
 
-        if (HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_CPT(dev_priv)) {
                 /* Workaround: Clear the timing override chicken bit again. */
                 reg = TRANS_CHICKEN2(pipe);
                 val = I915_READ(reg);
@@ -1926,6 +1926,18 @@ void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
 }
 
+enum transcoder intel_crtc_pch_transcoder(struct intel_crtc *crtc)
+{
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+
+        WARN_ON(!crtc->config->has_pch_encoder);
+
+        if (HAS_PCH_LPT(dev_priv))
+                return TRANSCODER_A;
+        else
+                return (enum transcoder) crtc->pipe;
+}
+
 /**
  * intel_enable_pipe - enable a pipe, asserting requirements
  * @crtc: crtc responsible for the pipe
@@ -1939,7 +1951,6 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
         struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = crtc->pipe;
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
-        enum pipe pch_transcoder;
         i915_reg_t reg;
         u32 val;
 
@@ -1949,11 +1960,6 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
         assert_cursor_disabled(dev_priv, pipe);
         assert_sprites_disabled(dev_priv, pipe);
 
-        if (HAS_PCH_LPT(dev_priv))
-                pch_transcoder = TRANSCODER_A;
-        else
-                pch_transcoder = pipe;
-
         /*
          * A pipe without a PLL won't actually be able to drive bits from
          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
@@ -1967,7 +1973,8 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
         } else {
                 if (crtc->config->has_pch_encoder) {
                         /* if driving the PCH, we need FDI enabled */
-                        assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
+                        assert_fdi_rx_pll_enabled(dev_priv,
+                                                  (enum pipe) intel_crtc_pch_transcoder(crtc));
                         assert_fdi_tx_pll_enabled(dev_priv,
                                                   (enum pipe) cpu_transcoder);
                 }
@@ -3033,7 +3040,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
                            ((crtc_state->pipe_src_h - 1) << 16) |
                            (crtc_state->pipe_src_w - 1));
                 I915_WRITE(DSPPOS(plane), 0);
-        } else if (IS_CHERRYVIEW(dev) && plane == PLANE_B) {
+        } else if (IS_CHERRYVIEW(dev_priv) && plane == PLANE_B) {
                 I915_WRITE(PRIMSIZE(plane),
                            ((crtc_state->pipe_src_h - 1) << 16) |
                            (crtc_state->pipe_src_w - 1));
@@ -3071,7 +3078,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
             fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
                 dspcntr |= DISPPLANE_TILED;
 
-        if (IS_G4X(dev))
+        if (IS_G4X(dev_priv))
                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
         intel_add_fb_offsets(&x, &y, plane_state, 0);
@@ -3144,7 +3151,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
         dspcntr = DISPPLANE_GAMMA_ENABLE;
         dspcntr |= DISPLAY_PLANE_ENABLE;
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
 
         switch (fb->pixel_format) {
@@ -3173,7 +3180,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
         if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
                 dspcntr |= DISPPLANE_TILED;
 
-        if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
+        if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv))
                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
         intel_add_fb_offsets(&x, &y, plane_state, 0);
@@ -3184,7 +3191,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
         if (rotation == DRM_ROTATE_180) {
                 dspcntr |= DISPPLANE_ROTATE_180;
 
-                if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
+                if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
                         x += (crtc_state->pipe_src_w - 1);
                         y += (crtc_state->pipe_src_h - 1);
                 }
@@ -3201,7 +3208,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
         I915_WRITE(DSPSURF(plane),
                    intel_fb_gtt_offset(fb, rotation) +
                    intel_crtc->dspaddr_offset);
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
         } else {
                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
@@ -3378,6 +3385,8 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_framebuffer *fb = plane_state->base.fb;
         const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
+        const struct skl_plane_wm *p_wm =
+                &crtc_state->wm.skl.optimal.planes[0];
         int pipe = intel_crtc->pipe;
         u32 plane_ctl;
         unsigned int rotation = plane_state->base.rotation;
@@ -3408,11 +3417,13 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
         dst_w--;
         dst_h--;
 
+        intel_crtc->dspaddr_offset = surf_addr;
+
         intel_crtc->adjusted_x = src_x;
         intel_crtc->adjusted_y = src_y;
 
         if (wm->dirty_pipes & drm_crtc_mask(&intel_crtc->base))
-                skl_write_plane_wm(intel_crtc, wm, 0);
+                skl_write_plane_wm(intel_crtc, p_wm, &wm->ddb, 0);
 
         I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
         I915_WRITE(PLANE_OFFSET(pipe, 0), (src_y << 16) | src_x);
@@ -3446,6 +3457,8 @@ static void skylake_disable_primary_plane(struct drm_plane *primary,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
+        const struct skl_plane_wm *p_wm = &cstate->wm.skl.optimal.planes[0];
         int pipe = intel_crtc->pipe;
 
         /*
@@ -3453,7 +3466,8 @@ static void skylake_disable_primary_plane(struct drm_plane *primary,
          * plane's visiblity isn't actually changing neither is its watermarks.
          */
         if (!crtc->primary->state->visible)
-                skl_write_plane_wm(intel_crtc, &dev_priv->wm.skl_results, 0);
+                skl_write_plane_wm(intel_crtc, p_wm,
+                                   &dev_priv->wm.skl_results.ddb, 0);
 
         I915_WRITE(PLANE_CTL(pipe, 0), 0);
         I915_WRITE(PLANE_SURF(pipe, 0), 0);
@@ -3601,8 +3615,6 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
 
         dev_priv->modeset_restore_state = NULL;
 
-        dev_priv->modeset_restore_state = NULL;
-
         /* reset doesn't touch the display */
         if (!gpu_reset_clobbers_display(dev_priv)) {
                 if (!state) {
@@ -3629,6 +3641,7 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
                 intel_runtime_pm_disable_interrupts(dev_priv);
                 intel_runtime_pm_enable_interrupts(dev_priv);
 
+                intel_pps_unlock_regs_wa(dev_priv);
                 intel_modeset_init_hw(dev);
 
                 spin_lock_irq(&dev_priv->irq_lock);
@@ -3713,7 +3726,7 @@ static void intel_update_pipe_config(struct intel_crtc *crtc,
 
                 if (pipe_config->pch_pfit.enabled)
                         skylake_pfit_enable(crtc);
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
                 if (pipe_config->pch_pfit.enabled)
                         ironlake_pfit_enable(crtc);
                 else if (old_crtc_state->pch_pfit.enabled)
@@ -3733,7 +3746,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
         /* enable normal train */
         reg = FDI_TX_CTL(pipe);
         temp = I915_READ(reg);
-        if (IS_IVYBRIDGE(dev)) {
+        if (IS_IVYBRIDGE(dev_priv)) {
                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
         } else {
@@ -3744,7 +3757,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
 
         reg = FDI_RX_CTL(pipe);
         temp = I915_READ(reg);
-        if (HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_CPT(dev_priv)) {
                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
         } else {
@@ -3758,7 +3771,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
         udelay(1000);
 
         /* IVB wants error correction enabled */
-        if (IS_IVYBRIDGE(dev))
+        if (IS_IVYBRIDGE(dev_priv))
                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
                            FDI_FE_ERRC_ENABLE);
 }
@@ -3902,7 +3915,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 
         reg = FDI_RX_CTL(pipe);
         temp = I915_READ(reg);
-        if (HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_CPT(dev_priv)) {
                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
         } else {
@@ -3946,7 +3959,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
         temp = I915_READ(reg);
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_2;
-        if (IS_GEN6(dev)) {
+        if (IS_GEN6(dev_priv)) {
                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
                 /* SNB-B */
                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
@@ -3955,7 +3968,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 
         reg = FDI_RX_CTL(pipe);
         temp = I915_READ(reg);
-        if (HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_CPT(dev_priv)) {
                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
         } else {
@@ -4209,7 +4222,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
         udelay(100);
 
         /* Ironlake workaround, disable clock pointer after downing FDI */
-        if (HAS_PCH_IBX(dev))
+        if (HAS_PCH_IBX(dev_priv))
                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
 
         /* still set train pattern 1 */
@@ -4221,7 +4234,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
 
         reg = FDI_RX_CTL(pipe);
         temp = I915_READ(reg);
-        if (HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_CPT(dev_priv)) {
                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
         } else {
@@ -4544,7 +4557,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 
         assert_pch_transcoder_disabled(dev_priv, pipe);
 
-        if (IS_IVYBRIDGE(dev))
+        if (IS_IVYBRIDGE(dev_priv))
                 ivybridge_update_fdi_bc_bifurcation(intel_crtc);
 
         /* Write the TU size bits before fdi link training, so that error
@@ -4557,7 +4570,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 
         /* We need to program the right clock selection before writing the pixel
          * mutliplier into the DPLL. */
-        if (HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_CPT(dev_priv)) {
                 u32 sel;
 
                 temp = I915_READ(PCH_DPLL_SEL);
@@ -4587,7 +4600,8 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
         intel_fdi_normal_train(crtc);
 
         /* For PCH DP, enable TRANS_DP_CTL */
-        if (HAS_PCH_CPT(dev) && intel_crtc_has_dp_encoder(intel_crtc->config)) {
+        if (HAS_PCH_CPT(dev_priv) &&
+            intel_crtc_has_dp_encoder(intel_crtc->config)) {
                 const struct drm_display_mode *adjusted_mode =
                         &intel_crtc->config->base.adjusted_mode;
                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
@@ -4857,7 +4871,7 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc)
                  * as some pre-programmed values are broken,
                  * e.g. x201.
                  */
-                if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+                if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
                                                  PF_PIPE_SEL_IVB(pipe));
                 else
@@ -4882,7 +4896,7 @@ void hsw_enable_ips(struct intel_crtc *crtc)
          */
 
         assert_plane_enabled(dev_priv, crtc->plane);
-        if (IS_BROADWELL(dev)) {
+        if (IS_BROADWELL(dev_priv)) {
                 mutex_lock(&dev_priv->rps.hw_lock);
                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
                 mutex_unlock(&dev_priv->rps.hw_lock);
@@ -4914,7 +4928,7 @@ void hsw_disable_ips(struct intel_crtc *crtc)
                 return;
 
         assert_plane_enabled(dev_priv, crtc->plane);
-        if (IS_BROADWELL(dev)) {
+        if (IS_BROADWELL(dev_priv)) {
                 mutex_lock(&dev_priv->rps.hw_lock);
                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
                 mutex_unlock(&dev_priv->rps.hw_lock);
@@ -4983,7 +4997,7 @@ intel_post_enable_primary(struct drm_crtc *crtc)
          * FIXME: Need to fix the logic to work when we turn off all planes
          * but leave the pipe running.
          */
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
         /* Underruns don't always raise interrupts, so check manually. */
@@ -5006,7 +5020,7 @@ intel_pre_disable_primary(struct drm_crtc *crtc)
          * FIXME: Need to fix the logic to work when we turn off all planes
          * but leave the pipe running.
          */
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 
         /*
@@ -5038,7 +5052,7 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
          * event which is after the vblank start event, so we need to have a
          * wait-for-vblank between disabling the plane and the pipe.
          */
-        if (HAS_GMCH_DISPLAY(dev)) {
+        if (HAS_GMCH_DISPLAY(dev_priv)) {
                 intel_set_memory_cxsr(dev_priv, false);
                 dev_priv->wm.vlv.cxsr = false;
                 intel_wait_for_vblank(dev, pipe);
@@ -5103,7 +5117,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
                         intel_pre_disable_primary(&crtc->base);
         }
 
-        if (pipe_config->disable_cxsr && HAS_GMCH_DISPLAY(dev)) {
+        if (pipe_config->disable_cxsr && HAS_GMCH_DISPLAY(dev_priv)) {
                 crtc->wm.cxsr_allowed = false;
 
                 /*
@@ -5381,7 +5395,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 
         intel_encoders_enable(crtc, pipe_config, old_state);
 
-        if (HAS_PCH_CPT(dev))
+        if (HAS_PCH_CPT(dev_priv))
                 cpt_verify_modeset(dev, intel_crtc->pipe);
 
         /* Must wait for vblank to avoid spurious PCH FIFO underruns */
@@ -5394,7 +5408,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 /* IPS only exists on ULT machines and is tied to pipe A. */
 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
 {
-        return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
+        return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
 }
 
 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
@@ -5506,7 +5520,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
         /* If we change the relative order between pipe/planes enabling, we need
          * to change the workaround. */
         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
-        if (IS_HASWELL(dev) && hsw_workaround_pipe != INVALID_PIPE) {
+        if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
         }
@@ -5563,7 +5577,7 @@ static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
         if (intel_crtc->config->has_pch_encoder) {
                 ironlake_disable_pch_transcoder(dev_priv, pipe);
 
-                if (HAS_PCH_CPT(dev)) {
+                if (HAS_PCH_CPT(dev_priv)) {
                         i915_reg_t reg;
                         u32 temp;
 
@@ -5697,13 +5711,13 @@ static enum intel_display_power_domain port_to_aux_power_domain(enum port port)
 enum intel_display_power_domain
 intel_display_port_power_domain(struct intel_encoder *intel_encoder)
 {
-        struct drm_device *dev = intel_encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
         struct intel_digital_port *intel_dig_port;
 
         switch (intel_encoder->type) {
         case INTEL_OUTPUT_UNKNOWN:
                 /* Only DDI platforms should ever use this output type */
-                WARN_ON_ONCE(!HAS_DDI(dev));
+                WARN_ON_ONCE(!HAS_DDI(dev_priv));
         case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_HDMI:
         case INTEL_OUTPUT_EDP:
@@ -5724,7 +5738,7 @@ intel_display_port_power_domain(struct intel_encoder *intel_encoder)
 enum intel_display_power_domain
 intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
 {
-        struct drm_device *dev = intel_encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
         struct intel_digital_port *intel_dig_port;
 
         switch (intel_encoder->type) {
@@ -5737,7 +5751,7 @@ intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
                  * what's the status of the given connectors, play safe and
                  * run the DP detection too.
                  */
-                WARN_ON_ONCE(!HAS_DDI(dev));
+                WARN_ON_ONCE(!HAS_DDI(dev_priv));
         case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_EDP:
                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
@@ -5833,7 +5847,7 @@ static void intel_update_max_cdclk(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
                 int max_cdclk, vco;
 
@@ -5855,9 +5869,9 @@ static void intel_update_max_cdclk(struct drm_device *dev)
                         max_cdclk = 308571;
 
                 dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
-        } else if (IS_BROXTON(dev)) {
+        } else if (IS_BROXTON(dev_priv)) {
                 dev_priv->max_cdclk_freq = 624000;
-        } else if (IS_BROADWELL(dev))  {
+        } else if (IS_BROADWELL(dev_priv))  {
                 /*
                  * FIXME with extra cooling we can allow
                  * 540 MHz for ULX and 675 Mhz for ULT.
@@ -5866,15 +5880,15 @@ static void intel_update_max_cdclk(struct drm_device *dev)
                  */
                 if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
                         dev_priv->max_cdclk_freq = 450000;
-                else if (IS_BDW_ULX(dev))
+                else if (IS_BDW_ULX(dev_priv))
                         dev_priv->max_cdclk_freq = 450000;
-                else if (IS_BDW_ULT(dev))
+                else if (IS_BDW_ULT(dev_priv))
                         dev_priv->max_cdclk_freq = 540000;
                 else
                         dev_priv->max_cdclk_freq = 675000;
-        } else if (IS_CHERRYVIEW(dev)) {
+        } else if (IS_CHERRYVIEW(dev_priv)) {
                 dev_priv->max_cdclk_freq = 320000;
-        } else if (IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv)) {
                 dev_priv->max_cdclk_freq = 400000;
         } else {
                 /* otherwise assume cdclk is fixed */
@@ -6674,7 +6688,7 @@ static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
          */
         intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
-        if (IS_CHERRYVIEW(dev))
+        if (IS_CHERRYVIEW(dev_priv))
                 cherryview_set_cdclk(dev, req_cdclk);
         else
                 valleyview_set_cdclk(dev, req_cdclk);
@@ -6702,7 +6716,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
         intel_set_pipe_timings(intel_crtc);
         intel_set_pipe_src_size(intel_crtc);
 
-        if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) {
+        if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
                 struct drm_i915_private *dev_priv = to_i915(dev);
 
                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
@@ -6717,7 +6731,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
 
         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
 
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(dev_priv)) {
                 chv_prepare_pll(intel_crtc, intel_crtc->config);
                 chv_enable_pll(intel_crtc, intel_crtc->config);
         } else {
@@ -6773,7 +6787,7 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
 
         intel_crtc->active = true;
 
-        if (!IS_GEN2(dev))
+        if (!IS_GEN2(dev_priv))
                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
         intel_encoders_pre_enable(crtc, pipe_config, old_state);
@@ -6821,7 +6835,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
          * On gen2 planes are double buffered but the pipe isn't, so we must
          * wait for planes to fully turn off before disabling the pipe.
          */
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 intel_wait_for_vblank(dev, pipe);
 
         intel_encoders_disable(crtc, old_crtc_state, old_state);
@@ -6836,9 +6850,9 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
 
         if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) {
-                if (IS_CHERRYVIEW(dev))
+                if (IS_CHERRYVIEW(dev_priv))
                         chv_disable_pll(dev_priv, pipe);
-                else if (IS_VALLEYVIEW(dev))
+                else if (IS_VALLEYVIEW(dev_priv))
                         vlv_disable_pll(dev_priv, pipe);
                 else
                         i9xx_disable_pll(intel_crtc);
@@ -6846,7 +6860,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
 
         intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
 
-        if (!IS_GEN2(dev))
+        if (!IS_GEN2(dev_priv))
                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 }
 
@@ -7026,6 +7040,7 @@ static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
                                      struct intel_crtc_state *pipe_config)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_atomic_state *state = pipe_config->base.state;
         struct intel_crtc *other_crtc;
         struct intel_crtc_state *other_crtc_state;
@@ -7038,7 +7053,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
                 return -EINVAL;
         }
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 if (pipe_config->fdi_lanes > 2) {
                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
                                       pipe_config->fdi_lanes);
@@ -7223,11 +7238,11 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
          */
-        if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
+        if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
                 return -EINVAL;
 
-        if (HAS_IPS(dev))
+        if (HAS_IPS(dev_priv))
                 hsw_compute_ips_config(crtc, pipe_config);
 
         if (pipe_config->has_pch_encoder)
@@ -7365,7 +7380,7 @@ static int haswell_get_display_clock_speed(struct drm_device *dev)
                 return 450000;
         else if (freq == LCPLL_CLK_FREQ_450)
                 return 450000;
-        else if (IS_HSW_ULT(dev))
+        else if (IS_HSW_ULT(dev_priv))
                 return 337500;
         else
                 return 540000;
@@ -7535,9 +7550,9 @@ static unsigned int intel_hpll_vco(struct drm_device *dev)
         uint8_t tmp = 0;
 
         /* FIXME other chipsets? */
-        if (IS_GM45(dev))
+        if (IS_GM45(dev_priv))
                 vco_table = ctg_vco;
-        else if (IS_G4X(dev))
+        else if (IS_G4X(dev_priv))
                 vco_table = elk_vco;
         else if (IS_CRESTLINE(dev))
                 vco_table = cl_vco;
@@ -7802,8 +7817,8 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
                  * for gen < 8) and if DRRS is supported (to make sure the
                  * registers are not unnecessarily accessed).
                  */
-                if (m2_n2 && (IS_CHERRYVIEW(dev) || INTEL_INFO(dev)->gen < 8) &&
-                        crtc->config->has_drrs) {
+                if (m2_n2 && (IS_CHERRYVIEW(dev_priv) ||
+                    INTEL_GEN(dev_priv) < 8) && crtc->config->has_drrs) {
                         I915_WRITE(PIPE_DATA_M2(transcoder),
                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
@@ -8105,7 +8120,7 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
         pipe_config->pixel_multiplier = 1;
         pipe_config->dpll = *dpll;
 
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(to_i915(dev))) {
                 chv_compute_dpll(crtc, pipe_config);
                 chv_prepare_pll(crtc, pipe_config);
                 chv_enable_pll(crtc, pipe_config);
@@ -8130,7 +8145,7 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
  */
 void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe)
 {
-        if (IS_CHERRYVIEW(dev))
+        if (IS_CHERRYVIEW(to_i915(dev)))
                 chv_disable_pll(to_i915(dev), pipe);
         else
                 vlv_disable_pll(to_i915(dev), pipe);
@@ -8154,7 +8169,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
         else
                 dpll |= DPLLB_MODE_DAC_SERIAL;
 
-        if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+        if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) || IS_G33(dev_priv)) {
                 dpll |= (crtc_state->pixel_multiplier - 1)
                         << SDVO_MULTIPLIER_SHIFT_HIRES;
         }
@@ -8171,7 +8186,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
         else {
                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
-                if (IS_G4X(dev) && reduced_clock)
+                if (IS_G4X(dev_priv) && reduced_clock)
                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
         }
         switch (clock->p2) {
@@ -8233,7 +8248,8 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
                         dpll |= PLL_P2_DIVIDE_BY_4;
         }
 
-        if (!IS_I830(dev) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
+        if (!IS_I830(dev_priv) &&
+            intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
                 dpll |= DPLL_DVO_2X_MODE;
 
         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
@@ -8302,7 +8318,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
          * documented on the DDI_FUNC_CTL register description, EDP Input Select
          * bits. */
-        if (IS_HASWELL(dev) && cpu_transcoder == TRANSCODER_EDP &&
+        if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
             (pipe == PIPE_B || pipe == PIPE_C))
                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
 
@@ -8412,7 +8428,8 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
                 pipeconf |= PIPECONF_DOUBLE_WIDE;
 
         /* only g4x and later have fancy bpc/dither controls */
-        if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
+            IS_CHERRYVIEW(dev_priv)) {
                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
                 if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
                         pipeconf |= PIPECONF_DITHER_EN |
@@ -8452,7 +8469,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
         } else
                 pipeconf |= PIPECONF_PROGRESSIVE;
 
-        if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+        if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
              intel_crtc->config->limited_color_range)
                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
 
@@ -8656,7 +8673,8 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc,
         struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t tmp;
 
-        if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
+        if (INTEL_GEN(dev_priv) <= 3 &&
+            (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
                 return;
 
         tmp = I915_READ(PFIT_CONTROL);
@@ -8828,7 +8846,8 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
         if (!(tmp & PIPECONF_ENABLE))
                 goto out;
 
-        if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
+            IS_CHERRYVIEW(dev_priv)) {
                 switch (tmp & PIPECONF_BPC_MASK) {
                 case PIPECONF_6BPC:
                         pipe_config->pipe_bpp = 18;
@@ -8844,7 +8863,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                 }
         }
 
-        if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+        if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
             (tmp & PIPECONF_COLOR_RANGE_SELECT))
                 pipe_config->limited_color_range = true;
 
@@ -8858,7 +8877,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 
         if (INTEL_INFO(dev)->gen >= 4) {
                 /* No way to read it out on pipes B and C */
-                if (IS_CHERRYVIEW(dev) && crtc->pipe != PIPE_A)
+                if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
                 else
                         tmp = I915_READ(DPLL_MD(crtc->pipe));
@@ -8866,7 +8885,8 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
                 pipe_config->dpll_hw_state.dpll_md = tmp;
-        } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+        } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
+                   IS_G33(dev_priv)) {
                 tmp = I915_READ(DPLL(crtc->pipe));
                 pipe_config->pixel_multiplier =
                         ((tmp & SDVO_MULTIPLIER_MASK)
@@ -8878,13 +8898,13 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                 pipe_config->pixel_multiplier = 1;
         }
         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
-        if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
+        if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
                 /*
                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
                  * on 830. Filter it out here so that we don't
                  * report errors due to that.
                  */
-                if (IS_I830(dev))
+                if (IS_I830(dev_priv))
                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
 
                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
@@ -8896,9 +8916,9 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                                                      DPLL_PORTB_READY_MASK);
         }
 
-        if (IS_CHERRYVIEW(dev))
+        if (IS_CHERRYVIEW(dev_priv))
                 chv_crtc_clock_get(crtc, pipe_config);
-        else if (IS_VALLEYVIEW(dev))
+        else if (IS_VALLEYVIEW(dev_priv))
                 vlv_crtc_clock_get(crtc, pipe_config);
         else
                 i9xx_crtc_clock_get(crtc, pipe_config);
@@ -8949,7 +8969,7 @@ static void ironlake_init_pch_refclk(struct drm_device *dev)
                 }
         }
 
-        if (HAS_PCH_IBX(dev)) {
+        if (HAS_PCH_IBX(dev_priv)) {
                 has_ck505 = dev_priv->vbt.display_clock_mode;
                 can_ssc = has_ck505;
         } else {
@@ -9197,7 +9217,8 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
 
         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
                 with_spread = true;
-        if (WARN(HAS_PCH_LPT_LP(dev) && with_fdi, "LP PCH doesn't have FDI\n"))
+        if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
+            with_fdi, "LP PCH doesn't have FDI\n"))
                 with_fdi = false;
 
         mutex_lock(&dev_priv->sb_lock);
@@ -9220,7 +9241,7 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
                 }
         }
 
-        reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
+        reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
@@ -9236,7 +9257,7 @@ static void lpt_disable_clkout_dp(struct drm_device *dev)
 
         mutex_lock(&dev_priv->sb_lock);
 
-        reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
+        reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
@@ -9344,9 +9365,11 @@ static void lpt_init_pch_refclk(struct drm_device *dev)
  */
 void intel_init_pch_refclk(struct drm_device *dev)
 {
-        if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
                 ironlake_init_pch_refclk(dev);
-        else if (HAS_PCH_LPT(dev))
+        else if (HAS_PCH_LPT(dev_priv))
                 lpt_init_pch_refclk(dev);
 }
 
@@ -9475,7 +9498,7 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 if ((intel_panel_use_ssc(dev_priv) &&
                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
-                    (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
+                    (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev)))
                         factor = 25;
         } else if (crtc_state->sdvo_tv_clock)
                 factor = 20;
@@ -9511,6 +9534,24 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
         if (intel_crtc_has_dp_encoder(crtc_state))
                 dpll |= DPLL_SDVO_HIGH_SPEED;
 
+        /*
+         * The high speed IO clock is only really required for
+         * SDVO/HDMI/DP, but we also enable it for CRT to make it
+         * possible to share the DPLL between CRT and HDMI. Enabling
+         * the clock needlessly does no real harm, except use up a
+         * bit of power potentially.
+         *
+         * We'll limit this to IVB with 3 pipes, since it has only two
+         * DPLLs and so DPLL sharing is the only way to get three pipes
+         * driving PCH ports at the same time. On SNB we could do this,
+         * and potentially avoid enabling the second DPLL, but it's not
+         * clear if it''s a win or loss power wise. No point in doing
+         * this on ILK at all since it has a fixed DPLL<->pipe mapping.
+         */
+        if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
+            intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
+                dpll |= DPLL_SDVO_HIGH_SPEED;
+
         /* compute bitmask from p1 value */
         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
         /* also FPA1 */
@@ -9817,7 +9858,7 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc,
                 /* We currently do not free assignements of panel fitters on
                  * ivb/hsw (since we don't use the higher upscaling modes which
                  * differentiates them) so just WARN about this case for now. */
-                if (IS_GEN7(dev)) {
+                if (IS_GEN7(dev_priv)) {
                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
                                 PF_PIPE_SEL_IVB(crtc->pipe));
                 }
@@ -9862,7 +9903,7 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc,
         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
 
         base = I915_READ(DSPSURF(pipe)) & 0xfffff000;
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 offset = I915_READ(DSPOFFSET(pipe));
         } else {
                 if (plane_config->tiling)
@@ -10006,7 +10047,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
         I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n");
         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
              "CPU PWM1 enabled\n");
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev_priv))
                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
                      "CPU PWM2 enabled\n");
         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
@@ -10026,9 +10067,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 
 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = &dev_priv->drm;
-
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev_priv))
                 return I915_READ(D_COMP_HSW);
         else
                 return I915_READ(D_COMP_BDW);
@@ -10036,9 +10075,7 @@ static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
 
 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
 {
-        struct drm_device *dev = &dev_priv->drm;
-
-        if (IS_HASWELL(dev)) {
+        if (IS_HASWELL(dev_priv)) {
                 mutex_lock(&dev_priv->rps.hw_lock);
                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
                                             val))
@@ -10186,7 +10223,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv)
 
         DRM_DEBUG_KMS("Enabling package C8+\n");
 
-        if (HAS_PCH_LPT_LP(dev)) {
+        if (HAS_PCH_LPT_LP(dev_priv)) {
                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
@@ -10206,7 +10243,7 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv)
         hsw_restore_lcpll(dev_priv);
         lpt_init_pch_refclk(dev);
 
-        if (HAS_PCH_LPT_LP(dev)) {
+        if (HAS_PCH_LPT_LP(dev_priv)) {
                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
@@ -10630,9 +10667,9 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 
         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
         else
                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
@@ -10715,7 +10752,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
                         ironlake_get_pfit_config(crtc, pipe_config);
         }
 
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev_priv))
                 pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
                         (I915_READ(IPS_CTL) & IPS_ENABLE);
 
@@ -10803,12 +10840,15 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
         const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
+        const struct skl_plane_wm *p_wm =
+                &cstate->wm.skl.optimal.planes[PLANE_CURSOR];
         int pipe = intel_crtc->pipe;
         uint32_t cntl = 0;
 
         if (INTEL_GEN(dev_priv) >= 9 && wm->dirty_pipes & drm_crtc_mask(crtc))
-                skl_write_cursor_wm(intel_crtc, wm);
+                skl_write_cursor_wm(intel_crtc, p_wm, &wm->ddb);
 
         if (plane_state && plane_state->base.visible) {
                 cntl = MCURSOR_GAMMA_ENABLE;
@@ -10828,7 +10868,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
                 }
                 cntl |= pipe << 28; /* Connect to correct pipe */
 
-                if (HAS_DDI(dev))
+                if (HAS_DDI(dev_priv))
                         cntl |= CURSOR_PIPE_CSC_ENABLE;
 
                 if (plane_state->base.rotation == DRM_ROTATE_180)
@@ -10876,7 +10916,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
                 pos |= y << CURSOR_Y_SHIFT;
 
                 /* ILK+ do this automagically */
-                if (HAS_GMCH_DISPLAY(dev) &&
+                if (HAS_GMCH_DISPLAY(dev_priv) &&
                     plane_state->base.rotation == DRM_ROTATE_180) {
                         base += (plane_state->base.crtc_h *
                                  plane_state->base.crtc_w - 1) * 4;
@@ -10885,13 +10925,13 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 
         I915_WRITE(CURPOS(pipe), pos);
 
-        if (IS_845G(dev) || IS_I865G(dev))
+        if (IS_845G(dev_priv) || IS_I865G(dev_priv))
                 i845_update_cursor(crtc, base, plane_state);
         else
                 i9xx_update_cursor(crtc, base, plane_state);
 }
 
-static bool cursor_size_ok(struct drm_device *dev,
+static bool cursor_size_ok(struct drm_i915_private *dev_priv,
                            uint32_t width, uint32_t height)
 {
         if (width == 0 || height == 0)
@@ -10903,11 +10943,11 @@ static bool cursor_size_ok(struct drm_device *dev,
          * the precision of the register. Everything else requires
          * square cursors, limited to a few power-of-two sizes.
          */
-        if (IS_845G(dev) || IS_I865G(dev)) {
+        if (IS_845G(dev_priv) || IS_I865G(dev_priv)) {
                 if ((width & 63) != 0)
                         return false;
 
-                if (width > (IS_845G(dev) ? 64 : 512))
+                if (width > (IS_845G(dev_priv) ? 64 : 512))
                         return false;
 
                 if (height > 1023)
@@ -10916,7 +10956,7 @@ static bool cursor_size_ok(struct drm_device *dev,
                 switch (width | height) {
                 case 256:
                 case 128:
-                        if (IS_GEN2(dev))
+                        if (IS_GEN2(dev_priv))
                                 return false;
                 case 64:
                         break;
@@ -11299,9 +11339,9 @@ static int i9xx_pll_refclk(struct drm_device *dev,
 
         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
                 return dev_priv->vbt.lvds_ssc_freq;
-        else if (HAS_PCH_SPLIT(dev))
+        else if (HAS_PCH_SPLIT(dev_priv))
                 return 120000;
-        else if (!IS_GEN2(dev))
+        else if (!IS_GEN2(dev_priv))
                 return 96000;
         else
                 return 48000;
@@ -11334,7 +11374,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
         }
 
-        if (!IS_GEN2(dev)) {
+        if (!IS_GEN2(dev_priv)) {
                 if (IS_PINEVIEW(dev))
                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
@@ -11362,7 +11402,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
                 else
                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
         } else {
-                u32 lvds = IS_I830(dev) ? 0 : I915_READ(LVDS);
+                u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
 
                 if (is_lvds) {
@@ -11563,7 +11603,7 @@ static bool __pageflip_finished_cs(struct intel_crtc *crtc,
          * really needed there. But since ctg has the registers,
          * include it in the check anyway.
          */
-        if (INTEL_INFO(dev)->gen < 5 && !IS_G4X(dev))
+        if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
                 return true;
 
         /*
@@ -11833,6 +11873,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
                                  struct drm_i915_gem_request *req,
                                  uint32_t flags)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_ring *ring = req->ring;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         uint32_t plane_bit = 0;
@@ -11861,7 +11902,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
                  * 48bits addresses, and we need a NOOP for the batch size to
                  * stay even.
                  */
-                if (IS_GEN8(dev))
+                if (IS_GEN8(dev_priv))
                         len += 2;
         }
 
@@ -11898,7 +11939,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
                 intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
                                           DERRMR_PIPEB_PRI_FLIP_DONE |
                                           DERRMR_PIPEC_PRI_FLIP_DONE));
-                if (IS_GEN8(dev))
+                if (IS_GEN8(dev_priv))
                         intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8 |
                                               MI_SRM_LRM_GLOBAL_GTT);
                 else
@@ -11907,7 +11948,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
                 intel_ring_emit_reg(ring, DERRMR);
                 intel_ring_emit(ring,
                                 i915_ggtt_offset(req->engine->scratch) + 256);
-                if (IS_GEN8(dev)) {
+                if (IS_GEN8(dev_priv)) {
                         intel_ring_emit(ring, 0);
                         intel_ring_emit(ring, MI_NOOP);
                 }
@@ -12226,23 +12267,23 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
         atomic_inc(&intel_crtc->unpin_work_count);
 
-        if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
+        if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
                 work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
-                engine = &dev_priv->engine[BCS];
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+                engine = dev_priv->engine[BCS];
                 if (fb->modifier[0] != old_fb->modifier[0])
                         /* vlv: DISPLAY_FLIP fails to change tiling */
                         engine = NULL;
-        } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
-                engine = &dev_priv->engine[BCS];
+        } else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
+                engine = dev_priv->engine[BCS];
         } else if (INTEL_INFO(dev)->gen >= 7) {
                 engine = i915_gem_active_get_engine(&obj->last_write,
                                                     &obj->base.dev->struct_mutex);
                 if (engine == NULL || engine->id != RCS)
-                        engine = &dev_priv->engine[BCS];
+                        engine = dev_priv->engine[BCS];
         } else {
-                engine = &dev_priv->engine[RCS];
+                engine = dev_priv->engine[RCS];
         }
 
         mmio_flip = use_mmio_flip(engine, obj);
@@ -12273,7 +12314,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
                 work->flip_queued_req = i915_gem_active_get(&obj->last_write,
                                                             &obj->base.dev->struct_mutex);
-                schedule_work(&work->mmio_work);
+                queue_work(system_unbound_wq, &work->mmio_work);
         } else {
                 request = i915_gem_request_alloc(engine, engine->last_context);
                 if (IS_ERR(request)) {
@@ -12430,7 +12471,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
         struct drm_framebuffer *fb = plane_state->fb;
         int ret;
 
-        if (INTEL_GEN(dev) >= 9 && plane->type != DRM_PLANE_TYPE_CURSOR) {
+        if (INTEL_GEN(dev_priv) >= 9 && plane->type != DRM_PLANE_TYPE_CURSOR) {
                 ret = skl_update_scaler_plane(
                         to_intel_crtc_state(crtc_state),
                         to_intel_plane_state(plane_state));
@@ -12509,7 +12550,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
          * cstate->update_wm was already set above, so this flag will
          * take effect when we commit and program watermarks.
          */
-        if (plane->type == DRM_PLANE_TYPE_OVERLAY && IS_IVYBRIDGE(dev) &&
+        if (plane->type == DRM_PLANE_TYPE_OVERLAY && IS_IVYBRIDGE(dev_priv) &&
             needs_scaling(to_intel_plane_state(plane_state)) &&
             !needs_scaling(old_plane_state))
                 pipe_config->disable_lp_wm = true;
@@ -12685,15 +12726,16 @@ static int
 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
                           struct intel_crtc_state *pipe_config)
 {
-        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct drm_atomic_state *state;
         struct drm_connector *connector;
         struct drm_connector_state *connector_state;
         int bpp, i;
 
-        if ((IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)))
+        if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
+            IS_CHERRYVIEW(dev_priv)))
                 bpp = 10*3;
-        else if (INTEL_INFO(dev)->gen >= 5)
+        else if (INTEL_GEN(dev_priv) >= 5)
                 bpp = 12*3;
         else
                 bpp = 8*3;
@@ -12731,6 +12773,7 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
                                    const char *context)
 {
         struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_plane *plane;
         struct intel_plane *intel_plane;
         struct intel_plane_state *state;
@@ -12792,7 +12835,7 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
         DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled);
         DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
 
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 DRM_DEBUG_KMS("dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
                               "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
                               "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
@@ -12807,13 +12850,13 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
                               pipe_config->dpll_hw_state.pll9,
                               pipe_config->dpll_hw_state.pll10,
                               pipe_config->dpll_hw_state.pcsdw12);
-        } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 DRM_DEBUG_KMS("dpll_hw_state: "
                               "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
                               pipe_config->dpll_hw_state.ctrl1,
                               pipe_config->dpll_hw_state.cfgcr1,
                               pipe_config->dpll_hw_state.cfgcr2);
-        } else if (HAS_DDI(dev)) {
+        } else if (HAS_DDI(dev_priv)) {
                 DRM_DEBUG_KMS("dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
                               pipe_config->dpll_hw_state.wrpll,
                               pipe_config->dpll_hw_state.spll);
@@ -12891,7 +12934,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
                 switch (encoder->type) {
                         unsigned int port_mask;
                 case INTEL_OUTPUT_UNKNOWN:
-                        if (WARN_ON(!HAS_DDI(dev)))
+                        if (WARN_ON(!HAS_DDI(to_i915(dev))))
                                 break;
                 case INTEL_OUTPUT_DP:
                 case INTEL_OUTPUT_HDMI:
@@ -13177,6 +13220,7 @@ intel_pipe_config_compare(struct drm_device *dev,
                           struct intel_crtc_state *pipe_config,
                           bool adjust)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         bool ret = true;
 
 #define INTEL_ERR_OR_DBG_KMS(fmt, ...) \
@@ -13322,8 +13366,8 @@ intel_pipe_config_compare(struct drm_device *dev,
 
         PIPE_CONF_CHECK_I(pixel_multiplier);
         PIPE_CONF_CHECK_I(has_hdmi_sink);
-        if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
-            IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
+            IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 PIPE_CONF_CHECK_I(limited_color_range);
         PIPE_CONF_CHECK_I(has_infoframe);
 
@@ -13363,7 +13407,7 @@ intel_pipe_config_compare(struct drm_device *dev,
         }
 
         /* BDW+ don't expose a synchronous way to read the state */
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev_priv))
                 PIPE_CONF_CHECK_I(ips_enabled);
 
         PIPE_CONF_CHECK_I(double_wide);
@@ -13382,7 +13426,7 @@ intel_pipe_config_compare(struct drm_device *dev,
         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
         PIPE_CONF_CHECK_X(dsi_pll.div);
 
-        if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5)
+        if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
                 PIPE_CONF_CHECK_I(pipe_bpp);
 
         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
@@ -13423,30 +13467,65 @@ static void verify_wm_state(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct skl_ddb_allocation hw_ddb, *sw_ddb;
-        struct skl_ddb_entry *hw_entry, *sw_entry;
+        struct skl_pipe_wm hw_wm, *sw_wm;
+        struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
+        struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         const enum pipe pipe = intel_crtc->pipe;
-        int plane;
+        int plane, level, max_level = ilk_wm_max_level(dev_priv);
 
         if (INTEL_INFO(dev)->gen < 9 || !new_state->active)
                 return;
 
+        skl_pipe_wm_get_hw_state(crtc, &hw_wm);
+        sw_wm = &intel_crtc->wm.active.skl;
+
         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
         sw_ddb = &dev_priv->wm.skl_hw.ddb;
 
         /* planes */
         for_each_plane(dev_priv, pipe, plane) {
-                hw_entry = &hw_ddb.plane[pipe][plane];
-                sw_entry = &sw_ddb->plane[pipe][plane];
+                hw_plane_wm = &hw_wm.planes[plane];
+                sw_plane_wm = &sw_wm->planes[plane];
 
-                if (skl_ddb_entry_equal(hw_entry, sw_entry))
-                        continue;
+                /* Watermarks */
+                for (level = 0; level <= max_level; level++) {
+                        if (skl_wm_level_equals(&hw_plane_wm->wm[level],
+                                                &sw_plane_wm->wm[level]))
+                                continue;
 
-                DRM_ERROR("mismatch in DDB state pipe %c plane %d "
-                          "(expected (%u,%u), found (%u,%u))\n",
-                          pipe_name(pipe), plane + 1,
-                          sw_entry->start, sw_entry->end,
-                          hw_entry->start, hw_entry->end);
+                        DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+                                  pipe_name(pipe), plane + 1, level,
+                                  sw_plane_wm->wm[level].plane_en,
+                                  sw_plane_wm->wm[level].plane_res_b,
+                                  sw_plane_wm->wm[level].plane_res_l,
+                                  hw_plane_wm->wm[level].plane_en,
+                                  hw_plane_wm->wm[level].plane_res_b,
+                                  hw_plane_wm->wm[level].plane_res_l);
+                }
+
+                if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
+                                         &sw_plane_wm->trans_wm)) {
+                        DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+                                  pipe_name(pipe), plane + 1,
+                                  sw_plane_wm->trans_wm.plane_en,
+                                  sw_plane_wm->trans_wm.plane_res_b,
+                                  sw_plane_wm->trans_wm.plane_res_l,
+                                  hw_plane_wm->trans_wm.plane_en,
+                                  hw_plane_wm->trans_wm.plane_res_b,
+                                  hw_plane_wm->trans_wm.plane_res_l);
+                }
+
+                /* DDB */
+                hw_ddb_entry = &hw_ddb.plane[pipe][plane];
+                sw_ddb_entry = &sw_ddb->plane[pipe][plane];
+
+                if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
+                        DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
+                                  pipe_name(pipe), plane + 1,
+                                  sw_ddb_entry->start, sw_ddb_entry->end,
+                                  hw_ddb_entry->start, hw_ddb_entry->end);
+                }
         }
 
         /*
@@ -13456,15 +13535,46 @@ static void verify_wm_state(struct drm_crtc *crtc,
          * once the plane becomes visible, we can skip this check
          */
         if (intel_crtc->cursor_addr) {
-                hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
-                sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
+                hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
+                sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
+
+                /* Watermarks */
+                for (level = 0; level <= max_level; level++) {
+                        if (skl_wm_level_equals(&hw_plane_wm->wm[level],
+                                                &sw_plane_wm->wm[level]))
+                                continue;
+
+                        DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+                                  pipe_name(pipe), level,
+                                  sw_plane_wm->wm[level].plane_en,
+                                  sw_plane_wm->wm[level].plane_res_b,
+                                  sw_plane_wm->wm[level].plane_res_l,
+                                  hw_plane_wm->wm[level].plane_en,
+                                  hw_plane_wm->wm[level].plane_res_b,
+                                  hw_plane_wm->wm[level].plane_res_l);
+                }
+
+                if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
+                                         &sw_plane_wm->trans_wm)) {
+                        DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+                                  pipe_name(pipe),
+                                  sw_plane_wm->trans_wm.plane_en,
+                                  sw_plane_wm->trans_wm.plane_res_b,
+                                  sw_plane_wm->trans_wm.plane_res_l,
+                                  hw_plane_wm->trans_wm.plane_en,
+                                  hw_plane_wm->trans_wm.plane_res_b,
+                                  hw_plane_wm->trans_wm.plane_res_l);
+                }
 
-                if (!skl_ddb_entry_equal(hw_entry, sw_entry)) {
-                        DRM_ERROR("mismatch in DDB state pipe %c cursor "
-                                  "(expected (%u,%u), found (%u,%u))\n",
+                /* DDB */
+                hw_ddb_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
+                sw_ddb_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
+
+                if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
+                        DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
                                   pipe_name(pipe),
-                                  sw_entry->start, sw_entry->end,
-                                  hw_entry->start, hw_entry->end);
+                                  sw_ddb_entry->start, sw_ddb_entry->end,
+                                  hw_ddb_entry->start, hw_ddb_entry->end);
                 }
         }
 }
@@ -13715,7 +13825,7 @@ intel_modeset_verify_disabled(struct drm_device *dev)
 
 static void update_scanline_offset(struct intel_crtc *crtc)
 {
-        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
         /*
          * The scanline counter increments at the leading edge of hsync.
@@ -13735,7 +13845,7 @@ static void update_scanline_offset(struct intel_crtc *crtc)
          * there's an extra 1 line difference. So we need to add two instead of
          * one to the value.
          */
-        if (IS_GEN2(dev)) {
+        if (IS_GEN2(dev_priv)) {
                 const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
                 int vtotal;
 
@@ -13744,7 +13854,7 @@ static void update_scanline_offset(struct intel_crtc *crtc)
                         vtotal /= 2;
 
                 crtc->scanline_offset = vtotal - 1;
-        } else if (HAS_DDI(dev) &&
+        } else if (HAS_DDI(dev_priv) &&
                    intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) {
                 crtc->scanline_offset = 2;
         } else
@@ -14222,12 +14332,11 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
                              unsigned int *crtc_vblank_mask)
 {
         struct drm_device *dev = state->dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
         struct drm_crtc *crtc;
+        struct intel_crtc *intel_crtc;
         struct drm_crtc_state *old_crtc_state;
-        struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
-        struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
+        struct intel_crtc_state *cstate;
         unsigned int updated = 0;
         bool progress;
         enum pipe pipe;
@@ -14245,12 +14354,14 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
                 for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
                         bool vbl_wait = false;
                         unsigned int cmask = drm_crtc_mask(crtc);
-                        pipe = to_intel_crtc(crtc)->pipe;
+
+                        intel_crtc = to_intel_crtc(crtc);
+                        cstate = to_intel_crtc_state(crtc->state);
+                        pipe = intel_crtc->pipe;
 
                         if (updated & cmask || !crtc->state->active)
                                 continue;
-                        if (skl_ddb_allocation_overlaps(state, cur_ddb, new_ddb,
-                                                        pipe))
+                        if (skl_ddb_allocation_overlaps(state, intel_crtc))
                                 continue;
 
                         updated |= cmask;
@@ -14261,7 +14372,8 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
                          * then we need to wait for a vblank to pass for the
                          * new ddb allocation to take effect.
                          */
-                        if (!skl_ddb_allocation_equals(cur_ddb, new_ddb, pipe) &&
+                        if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb,
+                                                 &intel_crtc->hw_ddb) &&
                             !crtc->state->active_changed &&
                             intel_state->wm_results.dirty_pipes != updated)
                                 vbl_wait = true;
@@ -14369,8 +14481,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
                  * SKL workaround: bspec recommends we disable the SAGV when we
                  * have more then one pipe enabled
                  */
-                if (IS_SKYLAKE(dev_priv) && !skl_can_enable_sagv(state))
-                        skl_disable_sagv(dev_priv);
+                if (!intel_can_enable_sagv(state))
+                        intel_disable_sagv(dev_priv);
 
                 intel_modeset_verify_disabled(dev);
         }
@@ -14427,9 +14539,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
                 intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state);
         }
 
-        if (IS_SKYLAKE(dev_priv) && intel_state->modeset &&
-            skl_can_enable_sagv(state))
-                skl_enable_sagv(dev_priv);
+        if (intel_state->modeset && intel_can_enable_sagv(state))
+                intel_enable_sagv(dev_priv);
 
         drm_atomic_helper_commit_hw_done(state);
 
@@ -14642,6 +14753,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
                        struct drm_plane_state *new_state)
 {
         struct drm_device *dev = plane->dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_framebuffer *fb = new_state->fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
@@ -14693,7 +14805,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 
         if (plane->type == DRM_PLANE_TYPE_CURSOR &&
             INTEL_INFO(dev)->cursor_needs_physical) {
-                int align = IS_I830(dev) ? 16 * 1024 : 256;
+                int align = IS_I830(dev_priv) ? 16 * 1024 : 256;
                 ret = i915_gem_object_attach_phys(obj, align);
                 if (ret)
                         DRM_DEBUG_KMS("failed to attach phys object\n");
@@ -14818,6 +14930,8 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_crtc_state *intel_cstate =
+                to_intel_crtc_state(crtc->state);
         struct intel_crtc_state *old_intel_state =
                 to_intel_crtc_state(old_crtc_state);
         bool modeset = needs_modeset(crtc->state);
@@ -14834,13 +14948,13 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
                 intel_color_load_luts(crtc->state);
         }
 
-        if (to_intel_crtc_state(crtc->state)->update_pipe)
+        if (intel_cstate->update_pipe) {
                 intel_update_pipe_config(intel_crtc, old_intel_state);
-        else if (INTEL_GEN(dev_priv) >= 9) {
+        } else if (INTEL_GEN(dev_priv) >= 9) {
                 skl_detach_scalers(intel_crtc);
 
                 I915_WRITE(PIPE_WM_LINETIME(pipe),
-                           dev_priv->wm.skl_hw.wm_linetime[pipe]);
+                           intel_cstate->wm.skl.optimal.linetime);
         }
 }
 
@@ -14883,6 +14997,7 @@ const struct drm_plane_funcs intel_plane_funcs = {
 static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
                                                     int pipe)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *primary = NULL;
         struct intel_plane_state *state = NULL;
         const uint32_t *intel_primary_formats;
@@ -14918,7 +15033,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
 
                 primary->update_plane = skylake_update_primary_plane;
                 primary->disable_plane = skylake_disable_primary_plane;
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
                 intel_primary_formats = i965_primary_formats;
                 num_formats = ARRAY_SIZE(i965_primary_formats);
 
@@ -14944,7 +15059,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
                                                intel_primary_formats, num_formats,
                                                DRM_PLANE_TYPE_PRIMARY,
                                                "plane 1%c", pipe_name(pipe));
-        else if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
+        else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
                 ret = drm_universal_plane_init(dev, &primary->base, 0,
                                                &intel_plane_funcs,
                                                intel_primary_formats, num_formats,
@@ -14959,18 +15074,18 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
         if (ret)
                 goto fail;
 
-        if (INTEL_GEN(dev) >= 9) {
+        if (INTEL_GEN(dev_priv) >= 9) {
                 supported_rotations =
                         DRM_ROTATE_0 | DRM_ROTATE_90 |
                         DRM_ROTATE_180 | DRM_ROTATE_270;
-        } else if (INTEL_GEN(dev) >= 4) {
+        } else if (INTEL_GEN(dev_priv) >= 4) {
                 supported_rotations =
                         DRM_ROTATE_0 | DRM_ROTATE_180;
         } else {
                 supported_rotations = DRM_ROTATE_0;
         }
 
-        if (INTEL_GEN(dev) >= 4)
+        if (INTEL_GEN(dev_priv) >= 4)
                 drm_plane_create_rotation_property(&primary->base,
                                                    DRM_ROTATE_0,
                                                    supported_rotations);
@@ -15010,7 +15125,8 @@ intel_check_cursor_plane(struct drm_plane *plane,
                 return 0;
 
         /* Check for which cursor types we support */
-        if (!cursor_size_ok(plane->dev, state->base.crtc_w, state->base.crtc_h)) {
+        if (!cursor_size_ok(to_i915(plane->dev), state->base.crtc_w,
+                            state->base.crtc_h)) {
                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
                           state->base.crtc_w, state->base.crtc_h);
                 return -EINVAL;
@@ -15037,7 +15153,7 @@ intel_check_cursor_plane(struct drm_plane *plane,
          * display power well must be turned off and on again.
          * Refuse the put the cursor into that compromised position.
          */
-        if (IS_CHERRYVIEW(plane->dev) && pipe == PIPE_C &&
+        if (IS_CHERRYVIEW(to_i915(plane->dev)) && pipe == PIPE_C &&
             state->base.visible && state->base.crtc_x < 0) {
                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
                 return -EINVAL;
@@ -15081,6 +15197,7 @@ intel_update_cursor_plane(struct drm_plane *plane,
 static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
                                                    int pipe)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *cursor = NULL;
         struct intel_plane_state *state = NULL;
         int ret;
@@ -15112,7 +15229,7 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
         if (ret)
                 goto fail;
 
-        if (INTEL_GEN(dev) >= 4)
+        if (INTEL_GEN(dev_priv) >= 4)
                 drm_plane_create_rotation_property(&cursor->base,
                                                    DRM_ROTATE_0,
                                                    DRM_ROTATE_0 |
@@ -15285,7 +15402,7 @@ static bool has_edp_a(struct drm_device *dev)
         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
                 return false;
 
-        if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
+        if (IS_GEN5(dev_priv) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
                 return false;
 
         return true;
@@ -15298,17 +15415,18 @@ static bool intel_crt_present(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen >= 9)
                 return false;
 
-        if (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
+        if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
                 return false;
 
-        if (IS_CHERRYVIEW(dev))
+        if (IS_CHERRYVIEW(dev_priv))
                 return false;
 
-        if (HAS_PCH_LPT_H(dev) && I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
+        if (HAS_PCH_LPT_H(dev_priv) &&
+            I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
                 return false;
 
         /* DDI E can't be used if DDI A requires 4 lanes */
-        if (HAS_DDI(dev) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
+        if (HAS_DDI(dev_priv) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
                 return false;
 
         if (!dev_priv->vbt.int_crt_support)
@@ -15371,7 +15489,7 @@ static void intel_setup_outputs(struct drm_device *dev)
         if (intel_crt_present(dev))
                 intel_crt_init(dev);
 
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 /*
                  * FIXME: Broxton doesn't support port detection via the
                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
@@ -15382,7 +15500,7 @@ static void intel_setup_outputs(struct drm_device *dev)
                 intel_ddi_init(dev, PORT_C);
 
                 intel_dsi_init(dev);
-        } else if (HAS_DDI(dev)) {
+        } else if (HAS_DDI(dev_priv)) {
                 int found;
 
                 /*
@@ -15392,7 +15510,7 @@ static void intel_setup_outputs(struct drm_device *dev)
                  */
                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
                 /* WaIgnoreDDIAStrap: skl */
-                if (found || IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+                if (found || IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
                         intel_ddi_init(dev, PORT_A);
 
                 /* DDI B, C and D detection is indicated by the SFUSE_STRAP
@@ -15408,13 +15526,13 @@ static void intel_setup_outputs(struct drm_device *dev)
                 /*
                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
                  */
-                if ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
+                if ((IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) &&
                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
                         intel_ddi_init(dev, PORT_E);
 
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
                 int found;
                 dpd_is_edp = intel_dp_is_edp(dev, PORT_D);
 
@@ -15441,7 +15559,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
                         intel_dp_init(dev, PCH_DP_D, PORT_D);
-        } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 bool has_edp, has_port;
 
                 /*
@@ -15473,7 +15591,7 @@ static void intel_setup_outputs(struct drm_device *dev)
                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
                         intel_hdmi_init(dev, VLV_HDMIC, PORT_C);
 
-                if (IS_CHERRYVIEW(dev)) {
+                if (IS_CHERRYVIEW(dev_priv)) {
                         /*
                          * eDP not supported on port D,
                          * so no need to worry about it
@@ -15486,18 +15604,18 @@ static void intel_setup_outputs(struct drm_device *dev)
                 }
 
                 intel_dsi_init(dev);
-        } else if (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) {
+        } else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) {
                 bool found = false;
 
                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
                         DRM_DEBUG_KMS("probing SDVOB\n");
                         found = intel_sdvo_init(dev, GEN3_SDVOB, PORT_B);
-                        if (!found && IS_G4X(dev)) {
+                        if (!found && IS_G4X(dev_priv)) {
                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
                                 intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
                         }
 
-                        if (!found && IS_G4X(dev))
+                        if (!found && IS_G4X(dev_priv))
                                 intel_dp_init(dev, DP_B, PORT_B);
                 }
 
@@ -15510,18 +15628,17 @@ static void intel_setup_outputs(struct drm_device *dev)
 
                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
 
-                        if (IS_G4X(dev)) {
+                        if (IS_G4X(dev_priv)) {
                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
                                 intel_hdmi_init(dev, GEN4_HDMIC, PORT_C);
                         }
-                        if (IS_G4X(dev))
+                        if (IS_G4X(dev_priv))
                                 intel_dp_init(dev, DP_C, PORT_C);
                 }
 
-                if (IS_G4X(dev) &&
-                    (I915_READ(DP_D) & DP_DETECTED))
+                if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
                         intel_dp_init(dev, DP_D, PORT_D);
-        } else if (IS_GEN2(dev))
+        } else if (IS_GEN2(dev_priv))
                 intel_dvo_init(dev);
 
         if (SUPPORTS_TV(dev))
@@ -15592,10 +15709,10 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = {
 };
 
 static
-u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
-                         uint32_t pixel_format)
+u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv,
+                         uint64_t fb_modifier, uint32_t pixel_format)
 {
-        u32 gen = INTEL_INFO(dev)->gen;
+        u32 gen = INTEL_INFO(dev_priv)->gen;
 
         if (gen >= 9) {
                 int cpp = drm_format_plane_cpp(pixel_format, 0);
@@ -15604,7 +15721,8 @@ u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
                  *  pixels and 32K bytes."
                  */
                 return min(8192 * cpp, 32768);
-        } else if (gen >= 5 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
+        } else if (gen >= 5 && !IS_VALLEYVIEW(dev_priv) &&
+                   !IS_CHERRYVIEW(dev_priv)) {
                 return 32*1024;
         } else if (gen >= 4) {
                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
@@ -15691,7 +15809,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
                 return -EINVAL;
         }
 
-        pitch_limit = intel_fb_pitch_limit(dev, mode_cmd->modifier[0],
+        pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->modifier[0],
                                            mode_cmd->pixel_format);
         if (mode_cmd->pitches[0] > pitch_limit) {
                 DRM_DEBUG("%s pitch (%u) must be at less than %d\n",
@@ -15729,7 +15847,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
                 }
                 break;
         case DRM_FORMAT_ABGR8888:
-                if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
+                if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
                     INTEL_INFO(dev)->gen < 9) {
                         format_name = drm_get_format_name(mode_cmd->pixel_format);
                         DRM_DEBUG("unsupported pixel format: %s\n", format_name);
@@ -15748,7 +15866,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
                 }
                 break;
         case DRM_FORMAT_ABGR2101010:
-                if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
+                if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
                         format_name = drm_get_format_name(mode_cmd->pixel_format);
                         DRM_DEBUG("unsupported pixel format: %s\n", format_name);
                         kfree(format_name);
@@ -15815,12 +15933,6 @@ intel_user_framebuffer_create(struct drm_device *dev,
         return fb;
 }
 
-#ifndef CONFIG_DRM_FBDEV_EMULATION
-static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
-{
-}
-#endif
-
 static const struct drm_mode_config_funcs intel_mode_funcs = {
         .fb_create = intel_user_framebuffer_create,
         .output_poll_changed = intel_fbdev_output_poll_changed,
@@ -16201,7 +16313,7 @@ static void i915_disable_vga(struct drm_device *dev)
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct pci_dev *pdev = dev_priv->drm.pdev;
         u8 sr1;
-        i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
+        i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
 
         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
@@ -16340,7 +16452,7 @@ void intel_modeset_init(struct drm_device *dev)
          * BIOS isn't using it, don't assume it will work even if the VBT
          * indicates as much.
          */
-        if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
+        if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
                                             DREF_SSC1_ENABLE);
 
@@ -16352,10 +16464,10 @@ void intel_modeset_init(struct drm_device *dev)
                 }
         }
 
-        if (IS_GEN2(dev)) {
+        if (IS_GEN2(dev_priv)) {
                 dev->mode_config.max_width = 2048;
                 dev->mode_config.max_height = 2048;
-        } else if (IS_GEN3(dev)) {
+        } else if (IS_GEN3(dev_priv)) {
                 dev->mode_config.max_width = 4096;
                 dev->mode_config.max_height = 4096;
         } else {
@@ -16363,10 +16475,10 @@ void intel_modeset_init(struct drm_device *dev)
                 dev->mode_config.max_height = 8192;
         }
 
-        if (IS_845G(dev) || IS_I865G(dev)) {
-                dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
+        if (IS_845G(dev_priv) || IS_I865G(dev_priv)) {
+                dev->mode_config.cursor_width = IS_845G(dev_priv) ? 64 : 512;
                 dev->mode_config.cursor_height = 1023;
-        } else if (IS_GEN2(dev)) {
+        } else if (IS_GEN2(dev_priv)) {
                 dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
                 dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
         } else {
@@ -16572,7 +16684,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
         if (crtc->active && !intel_crtc_has_encoders(crtc))
                 intel_crtc_disable_noatomic(&crtc->base);
 
-        if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
+        if (crtc->active || HAS_GMCH_DISPLAY(dev_priv)) {
                 /*
                  * We start out with underrun reporting disabled to avoid races.
                  * For correct bookkeeping mark this on active crtcs.
@@ -16647,7 +16759,7 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 void i915_redisable_vga_power_on(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
-        i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
+        i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
 
         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
@@ -16885,11 +16997,11 @@ intel_modeset_setup_hw_state(struct drm_device *dev)
                 pll->on = false;
         }
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 vlv_wm_get_hw_state(dev);
-        else if (IS_GEN9(dev))
+        else if (IS_GEN9(dev_priv))
                 skl_wm_get_hw_state(dev);
-        else if (HAS_PCH_SPLIT(dev))
+        else if (HAS_PCH_SPLIT(dev_priv))
                 ilk_wm_get_hw_state(dev);
 
         for_each_intel_crtc(dev, crtc) {
@@ -17080,6 +17192,8 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
         return 0;
 }
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 struct intel_display_error_state {
 
         u32 power_well_driver;
@@ -17218,7 +17332,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
                 return;
 
         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
                            error->power_well_driver);
         for_each_pipe(dev_priv, i) {
@@ -17235,7 +17349,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
                 }
-                if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
+                if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
                 if (INTEL_INFO(dev)->gen >= 4) {
                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
@@ -17262,3 +17376,5 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
         }
 }
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 01bc67194dba..3c2293bd24bf 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -344,7 +344,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
         DP |= DP_PORT_WIDTH(1);
         DP |= DP_LINK_TRAIN_PAT_1;
 
-        if (IS_CHERRYVIEW(dev))
+        if (IS_CHERRYVIEW(dev_priv))
                 DP |= DP_PIPE_SELECT_CHV(pipe);
         else if (pipe == PIPE_B)
                 DP |= DP_PIPEB_SELECT;
@@ -356,10 +356,10 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
          * So enable temporarily it if it's not already enabled.
          */
         if (!pll_enabled) {
-                release_cl_override = IS_CHERRYVIEW(dev) &&
+                release_cl_override = IS_CHERRYVIEW(dev_priv) &&
                         !chv_phy_powergate_ch(dev_priv, phy, ch, true);
 
-                if (vlv_force_pll_on(dev, pipe, IS_CHERRYVIEW(dev) ?
+                if (vlv_force_pll_on(dev, pipe, IS_CHERRYVIEW(dev_priv) ?
                                      &chv_dpll[0].dpll : &vlv_dpll[0].dpll)) {
                         DRM_ERROR("Failed to force on pll for pipe %c!\n",
                                   pipe_name(pipe));
@@ -570,8 +570,8 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
         struct drm_device *dev = &dev_priv->drm;
         struct intel_encoder *encoder;
 
-        if (WARN_ON(!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
-                    !IS_BROXTON(dev)))
+        if (WARN_ON(!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
+                    !IS_BROXTON(dev_priv)))
                 return;
 
         /*
@@ -591,7 +591,7 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
                         continue;
 
                 intel_dp = enc_to_intel_dp(&encoder->base);
-                if (IS_BROXTON(dev))
+                if (IS_BROXTON(dev_priv))
                         intel_dp->pps_reset = true;
                 else
                         intel_dp->pps_pipe = INVALID_PIPE;
@@ -664,7 +664,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 
         pps_lock(intel_dp);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
                 i915_reg_t pp_ctrl_reg, pp_div_reg;
                 u32 pp_div;
@@ -692,7 +692,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
-        if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+        if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
             intel_dp->pps_pipe == INVALID_PIPE)
                 return false;
 
@@ -706,7 +706,7 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
-        if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+        if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
             intel_dp->pps_pipe == INVALID_PIPE)
                 return false;
 
@@ -821,15 +821,16 @@ static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
                                      uint32_t aux_clock_divider)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv =
+                        to_i915(intel_dig_port->base.base.dev);
         uint32_t precharge, timeout;
 
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev_priv))
                 precharge = 3;
         else
                 precharge = 5;
 
-        if (IS_BROADWELL(dev) && intel_dig_port->port == PORT_A)
+        if (IS_BROADWELL(dev_priv) && intel_dig_port->port == PORT_A)
                 timeout = DP_AUX_CH_CTL_TIME_OUT_600us;
         else
                 timeout = DP_AUX_CH_CTL_TIME_OUT_400us;
@@ -1108,8 +1109,46 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
         return ret;
 }
 
+static enum port intel_aux_port(struct drm_i915_private *dev_priv,
+                                enum port port)
+{
+        const struct ddi_vbt_port_info *info =
+                &dev_priv->vbt.ddi_port_info[port];
+        enum port aux_port;
+
+        if (!info->alternate_aux_channel) {
+                DRM_DEBUG_KMS("using AUX %c for port %c (platform default)\n",
+                              port_name(port), port_name(port));
+                return port;
+        }
+
+        switch (info->alternate_aux_channel) {
+        case DP_AUX_A:
+                aux_port = PORT_A;
+                break;
+        case DP_AUX_B:
+                aux_port = PORT_B;
+                break;
+        case DP_AUX_C:
+                aux_port = PORT_C;
+                break;
+        case DP_AUX_D:
+                aux_port = PORT_D;
+                break;
+        default:
+                MISSING_CASE(info->alternate_aux_channel);
+                aux_port = PORT_A;
+                break;
+        }
+
+        DRM_DEBUG_KMS("using AUX %c for port %c (VBT)\n",
+                      port_name(aux_port), port_name(port));
+
+        return aux_port;
+}
+
 static i915_reg_t g4x_aux_ctl_reg(struct drm_i915_private *dev_priv,
-                                       enum port port)
+                                  enum port port)
 {
         switch (port) {
         case PORT_B:
@@ -1123,7 +1162,7 @@ static i915_reg_t g4x_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t g4x_aux_data_reg(struct drm_i915_private *dev_priv,
-                                        enum port port, int index)
+                                   enum port port, int index)
 {
         switch (port) {
         case PORT_B:
@@ -1137,7 +1176,7 @@ static i915_reg_t g4x_aux_data_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t ilk_aux_ctl_reg(struct drm_i915_private *dev_priv,
-                                       enum port port)
+                                  enum port port)
 {
         switch (port) {
         case PORT_A:
@@ -1153,7 +1192,7 @@ static i915_reg_t ilk_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t ilk_aux_data_reg(struct drm_i915_private *dev_priv,
-                                        enum port port, int index)
+                                   enum port port, int index)
 {
         switch (port) {
         case PORT_A:
@@ -1168,36 +1207,9 @@ static i915_reg_t ilk_aux_data_reg(struct drm_i915_private *dev_priv,
         }
 }
 
-/*
- * On SKL we don't have Aux for port E so we rely
- * on VBT to set a proper alternate aux channel.
- */
-static enum port skl_porte_aux_port(struct drm_i915_private *dev_priv)
-{
-        const struct ddi_vbt_port_info *info =
-                &dev_priv->vbt.ddi_port_info[PORT_E];
-
-        switch (info->alternate_aux_channel) {
-        case DP_AUX_A:
-                return PORT_A;
-        case DP_AUX_B:
-                return PORT_B;
-        case DP_AUX_C:
-                return PORT_C;
-        case DP_AUX_D:
-                return PORT_D;
-        default:
-                MISSING_CASE(info->alternate_aux_channel);
-                return PORT_A;
-        }
-}
-
 static i915_reg_t skl_aux_ctl_reg(struct drm_i915_private *dev_priv,
-                                       enum port port)
+                                  enum port port)
 {
-        if (port == PORT_E)
-                port = skl_porte_aux_port(dev_priv);
-
         switch (port) {
         case PORT_A:
         case PORT_B:
@@ -1211,11 +1223,8 @@ static i915_reg_t skl_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t skl_aux_data_reg(struct drm_i915_private *dev_priv,
-                                        enum port port, int index)
+                                   enum port port, int index)
 {
-        if (port == PORT_E)
-                port = skl_porte_aux_port(dev_priv);
-
         switch (port) {
         case PORT_A:
         case PORT_B:
@@ -1229,7 +1238,7 @@ static i915_reg_t skl_aux_data_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t intel_aux_ctl_reg(struct drm_i915_private *dev_priv,
-                                         enum port port)
+                                    enum port port)
 {
         if (INTEL_INFO(dev_priv)->gen >= 9)
                 return skl_aux_ctl_reg(dev_priv, port);
@@ -1240,7 +1249,7 @@ static i915_reg_t intel_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t intel_aux_data_reg(struct drm_i915_private *dev_priv,
-                                          enum port port, int index)
+                                     enum port port, int index)
 {
         if (INTEL_INFO(dev_priv)->gen >= 9)
                 return skl_aux_data_reg(dev_priv, port, index);
@@ -1253,7 +1262,8 @@ static i915_reg_t intel_aux_data_reg(struct drm_i915_private *dev_priv,
 static void intel_aux_reg_init(struct intel_dp *intel_dp)
 {
         struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
-        enum port port = dp_to_dig_port(intel_dp)->port;
+        enum port port = intel_aux_port(dev_priv,
+                                        dp_to_dig_port(intel_dp)->port);
         int i;
 
         intel_dp->aux_ch_ctl_reg = intel_aux_ctl_reg(dev_priv, port);
@@ -1297,14 +1307,10 @@ intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates)
 bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = dig_port->base.base.dev;
-
-        /* WaDisableHBR2:skl */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_B0))
-                return false;
+        struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
 
-        if ((IS_HASWELL(dev) && !IS_HSW_ULX(dev)) || IS_BROADWELL(dev) ||
-            (INTEL_INFO(dev)->gen >= 9))
+        if ((IS_HASWELL(dev_priv) && !IS_HSW_ULX(dev_priv)) ||
+            IS_BROADWELL(dev_priv) || (INTEL_GEN(dev_priv) >= 9))
                 return true;
         else
                 return false;
@@ -1314,13 +1320,13 @@ static int
 intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
         int size;
 
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 *source_rates = bxt_rates;
                 size = ARRAY_SIZE(bxt_rates);
-        } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 *source_rates = skl_rates;
                 size = ARRAY_SIZE(skl_rates);
         } else {
@@ -1340,19 +1346,20 @@ intel_dp_set_clock(struct intel_encoder *encoder,
                    struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const struct dp_link_dpll *divisor = NULL;
         int i, count = 0;
 
-        if (IS_G4X(dev)) {
+        if (IS_G4X(dev_priv)) {
                 divisor = gen4_dpll;
                 count = ARRAY_SIZE(gen4_dpll);
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
                 divisor = pch_dpll;
                 count = ARRAY_SIZE(pch_dpll);
-        } else if (IS_CHERRYVIEW(dev)) {
+        } else if (IS_CHERRYVIEW(dev_priv)) {
                 divisor = chv_dpll;
                 count = ARRAY_SIZE(chv_dpll);
-        } else if (IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv)) {
                 divisor = vlv_dpll;
                 count = ARRAY_SIZE(vlv_dpll);
         }
@@ -1567,7 +1574,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
         max_clock = common_len - 1;
 
-        if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && port != PORT_A)
+        if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && port != PORT_A)
                 pipe_config->has_pch_encoder = true;
 
         pipe_config->has_drrs = false;
@@ -1584,7 +1591,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
                                 return ret;
                 }
 
-                if (HAS_GMCH_DISPLAY(dev))
+                if (HAS_GMCH_DISPLAY(dev_priv))
                         intel_gmch_panel_fitting(intel_crtc, pipe_config,
                                                  intel_connector->panel.fitting_mode);
                 else
@@ -1709,7 +1716,7 @@ found:
                 to_intel_atomic_state(pipe_config->base.state)->cdclk_pll_vco = vco;
         }
 
-        if (!HAS_DDI(dev))
+        if (!HAS_DDI(dev_priv))
                 intel_dp_set_clock(encoder, pipe_config);
 
         return true;
@@ -1767,7 +1774,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 
         /* Split out the IBX/CPU vs CPT settings */
 
-        if (IS_GEN7(dev) && port == PORT_A) {
+        if (IS_GEN7(dev_priv) && port == PORT_A) {
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                         intel_dp->DP |= DP_SYNC_HS_HIGH;
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
@@ -1778,7 +1785,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
                         intel_dp->DP |= DP_ENHANCED_FRAMING;
 
                 intel_dp->DP |= crtc->pipe << 29;
-        } else if (HAS_PCH_CPT(dev) && port != PORT_A) {
+        } else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) {
                 u32 trans_dp;
 
                 intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT;
@@ -1790,8 +1797,9 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
                         trans_dp &= ~TRANS_DP_ENH_FRAMING;
                 I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp);
         } else {
-                if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
-                    !IS_CHERRYVIEW(dev) && pipe_config->limited_color_range)
+                if (!HAS_PCH_SPLIT(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
+                    !IS_CHERRYVIEW(dev_priv) &&
+                    pipe_config->limited_color_range)
                         intel_dp->DP |= DP_COLOR_RANGE_16_235;
 
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
@@ -1803,7 +1811,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
                 if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
                         intel_dp->DP |= DP_ENHANCED_FRAMING;
 
-                if (IS_CHERRYVIEW(dev))
+                if (IS_CHERRYVIEW(dev_priv))
                         intel_dp->DP |= DP_PIPE_SELECT_CHV(crtc->pipe);
                 else if (crtc->pipe == PIPE_B)
                         intel_dp->DP |= DP_PIPEB_SELECT;
@@ -2112,7 +2120,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
 
         pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
         pp = ironlake_get_pp_control(intel_dp);
-        if (IS_GEN5(dev)) {
+        if (IS_GEN5(dev_priv)) {
                 /* ILK workaround: disable reset around power sequence */
                 pp &= ~PANEL_POWER_RESET;
                 I915_WRITE(pp_ctrl_reg, pp);
@@ -2120,7 +2128,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
         }
 
         pp |= PANEL_POWER_ON;
-        if (!IS_GEN5(dev))
+        if (!IS_GEN5(dev_priv))
                 pp |= PANEL_POWER_RESET;
 
         I915_WRITE(pp_ctrl_reg, pp);
@@ -2129,7 +2137,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
         wait_panel_on(intel_dp);
         intel_dp->last_power_on = jiffies;
 
-        if (IS_GEN5(dev)) {
+        if (IS_GEN5(dev_priv)) {
                 pp |= PANEL_POWER_RESET; /* restore panel reset bit */
                 I915_WRITE(pp_ctrl_reg, pp);
                 POSTING_READ(pp_ctrl_reg);
@@ -2442,9 +2450,9 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
         if (!(tmp & DP_PORT_EN))
                 goto out;
 
-        if (IS_GEN7(dev) && port == PORT_A) {
+        if (IS_GEN7(dev_priv) && port == PORT_A) {
                 *pipe = PORT_TO_PIPE_CPT(tmp);
-        } else if (HAS_PCH_CPT(dev) && port != PORT_A) {
+        } else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) {
                 enum pipe p;
 
                 for_each_pipe(dev_priv, p) {
@@ -2459,7 +2467,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
 
                 DRM_DEBUG_KMS("No pipe for dp port 0x%x found\n",
                               i915_mmio_reg_offset(intel_dp->output_reg));
-        } else if (IS_CHERRYVIEW(dev)) {
+        } else if (IS_CHERRYVIEW(dev_priv)) {
                 *pipe = DP_PORT_TO_PIPE_CHV(tmp);
         } else {
                 *pipe = PORT_TO_PIPE(tmp);
@@ -2487,7 +2495,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
 
         pipe_config->has_audio = tmp & DP_AUDIO_OUTPUT_ENABLE && port != PORT_A;
 
-        if (HAS_PCH_CPT(dev) && port != PORT_A) {
+        if (HAS_PCH_CPT(dev_priv) && port != PORT_A) {
                 u32 trans_dp = I915_READ(TRANS_DP_CTL(crtc->pipe));
 
                 if (trans_dp & TRANS_DP_HSYNC_ACTIVE_HIGH)
@@ -2513,8 +2521,8 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
 
         pipe_config->base.adjusted_mode.flags |= flags;
 
-        if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
-            !IS_CHERRYVIEW(dev) && tmp & DP_COLOR_RANGE_16_235)
+        if (!HAS_PCH_SPLIT(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
+            !IS_CHERRYVIEW(dev_priv) && tmp & DP_COLOR_RANGE_16_235)
                 pipe_config->limited_color_range = true;
 
         pipe_config->lane_count =
@@ -2634,7 +2642,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
                 DRM_DEBUG_KMS("Using DP training pattern TPS%d\n",
                               dp_train_pat & DP_TRAINING_PATTERN_MASK);
 
-        if (HAS_DDI(dev)) {
+        if (HAS_DDI(dev_priv)) {
                 uint32_t temp = I915_READ(DP_TP_CTL(port));
 
                 if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE)
@@ -2660,8 +2668,8 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
                 }
                 I915_WRITE(DP_TP_CTL(port), temp);
 
-        } else if ((IS_GEN7(dev) && port == PORT_A) ||
-                   (HAS_PCH_CPT(dev) && port != PORT_A)) {
+        } else if ((IS_GEN7(dev_priv) && port == PORT_A) ||
+                   (HAS_PCH_CPT(dev_priv) && port != PORT_A)) {
                 *DP &= ~DP_LINK_TRAIN_MASK_CPT;
 
                 switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
@@ -2681,7 +2689,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
                 }
 
         } else {
-                if (IS_CHERRYVIEW(dev))
+                if (IS_CHERRYVIEW(dev_priv))
                         *DP &= ~DP_LINK_TRAIN_MASK_CHV;
                 else
                         *DP &= ~DP_LINK_TRAIN_MASK;
@@ -2697,7 +2705,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
                         *DP |= DP_LINK_TRAIN_PAT_2;
                         break;
                 case DP_TRAINING_PATTERN_3:
-                        if (IS_CHERRYVIEW(dev)) {
+                        if (IS_CHERRYVIEW(dev_priv)) {
                                 *DP |= DP_LINK_TRAIN_PAT_3_CHV;
                         } else {
                                 DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n");
@@ -2747,7 +2755,7 @@ static void intel_enable_dp(struct intel_encoder *encoder,
 
         pps_lock(intel_dp);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 vlv_init_panel_power_sequencer(intel_dp);
 
         intel_dp_enable_port(intel_dp, pipe_config);
@@ -2758,10 +2766,10 @@ static void intel_enable_dp(struct intel_encoder *encoder,
 
         pps_unlock(intel_dp);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 unsigned int lane_mask = 0x0;
 
-                if (IS_CHERRYVIEW(dev))
+                if (IS_CHERRYVIEW(dev_priv))
                         lane_mask = intel_dp_unused_lane_mask(pipe_config->lane_count);
 
                 vlv_wait_port_ready(dev_priv, dp_to_dig_port(intel_dp),
@@ -2981,17 +2989,17 @@ intel_dp_voltage_max(struct intel_dp *intel_dp)
         struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = dp_to_dig_port(intel_dp)->port;
 
-        if (IS_BROXTON(dev))
+        if (IS_BROXTON(dev_priv))
                 return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
         else if (INTEL_INFO(dev)->gen >= 9) {
                 if (dev_priv->vbt.edp.low_vswing && port == PORT_A)
                         return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
                 return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
-        } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
-        else if (IS_GEN7(dev) && port == PORT_A)
+        else if (IS_GEN7(dev_priv) && port == PORT_A)
                 return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
-        else if (HAS_PCH_CPT(dev) && port != PORT_A)
+        else if (HAS_PCH_CPT(dev_priv) && port != PORT_A)
                 return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
         else
                 return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
@@ -3000,10 +3008,10 @@ intel_dp_voltage_max(struct intel_dp *intel_dp)
 uint8_t
 intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         enum port port = dp_to_dig_port(intel_dp)->port;
 
-        if (INTEL_INFO(dev)->gen >= 9) {
+        if (INTEL_GEN(dev_priv) >= 9) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
                 case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         return DP_TRAIN_PRE_EMPH_LEVEL_3;
@@ -3016,7 +3024,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
                 default:
                         return DP_TRAIN_PRE_EMPH_LEVEL_0;
                 }
-        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
                 case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         return DP_TRAIN_PRE_EMPH_LEVEL_3;
@@ -3028,7 +3036,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
                 default:
                         return DP_TRAIN_PRE_EMPH_LEVEL_0;
                 }
-        } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
                 case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         return DP_TRAIN_PRE_EMPH_LEVEL_3;
@@ -3040,7 +3048,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
                 default:
                         return DP_TRAIN_PRE_EMPH_LEVEL_0;
                 }
-        } else if (IS_GEN7(dev) && port == PORT_A) {
+        } else if (IS_GEN7(dev_priv) && port == PORT_A) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
                 case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         return DP_TRAIN_PRE_EMPH_LEVEL_2;
@@ -3341,21 +3349,21 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp)
         uint32_t signal_levels, mask = 0;
         uint8_t train_set = intel_dp->train_set[0];
 
-        if (HAS_DDI(dev)) {
+        if (HAS_DDI(dev_priv)) {
                 signal_levels = ddi_signal_levels(intel_dp);
 
-                if (IS_BROXTON(dev))
+                if (IS_BROXTON(dev_priv))
                         signal_levels = 0;
                 else
                         mask = DDI_BUF_EMP_MASK;
-        } else if (IS_CHERRYVIEW(dev)) {
+        } else if (IS_CHERRYVIEW(dev_priv)) {
                 signal_levels = chv_signal_levels(intel_dp);
-        } else if (IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv)) {
                 signal_levels = vlv_signal_levels(intel_dp);
-        } else if (IS_GEN7(dev) && port == PORT_A) {
+        } else if (IS_GEN7(dev_priv) && port == PORT_A) {
                 signal_levels = gen7_edp_signal_levels(train_set);
                 mask = EDP_LINK_TRAIN_VOL_EMP_MASK_IVB;
-        } else if (IS_GEN6(dev) && port == PORT_A) {
+        } else if (IS_GEN6(dev_priv) && port == PORT_A) {
                 signal_levels = gen6_edp_signal_levels(train_set);
                 mask = EDP_LINK_TRAIN_VOL_EMP_MASK_SNB;
         } else {
@@ -3400,7 +3408,7 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
         enum port port = intel_dig_port->port;
         uint32_t val;
 
-        if (!HAS_DDI(dev))
+        if (!HAS_DDI(dev_priv))
                 return;
 
         val = I915_READ(DP_TP_CTL(port));
@@ -3435,7 +3443,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
         struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t DP = intel_dp->DP;
 
-        if (WARN_ON(HAS_DDI(dev)))
+        if (WARN_ON(HAS_DDI(dev_priv)))
                 return;
 
         if (WARN_ON((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0))
@@ -3443,12 +3451,12 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 
         DRM_DEBUG_KMS("\n");
 
-        if ((IS_GEN7(dev) && port == PORT_A) ||
-            (HAS_PCH_CPT(dev) && port != PORT_A)) {
+        if ((IS_GEN7(dev_priv) && port == PORT_A) ||
+            (HAS_PCH_CPT(dev_priv) && port != PORT_A)) {
                 DP &= ~DP_LINK_TRAIN_MASK_CPT;
                 DP |= DP_LINK_TRAIN_PAT_IDLE_CPT;
         } else {
-                if (IS_CHERRYVIEW(dev))
+                if (IS_CHERRYVIEW(dev_priv))
                         DP &= ~DP_LINK_TRAIN_MASK_CHV;
                 else
                         DP &= ~DP_LINK_TRAIN_MASK;
@@ -3466,7 +3474,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
          * to transcoder A after disabling it to allow the
          * matching HDMI port to be enabled on transcoder A.
          */
-        if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B && port != PORT_A) {
+        if (HAS_PCH_IBX(dev_priv) && crtc->pipe == PIPE_B && port != PORT_A) {
                 /*
                  * We get CPU/PCH FIFO underruns on the other pipe when
                  * doing the workaround. Sweep them under the rug.
@@ -3549,8 +3557,8 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
         /* Read the eDP Display control capabilities registers */
         if ((intel_dp->dpcd[DP_EDP_CONFIGURATION_CAP] & DP_DPCD_DISPLAY_CONTROL_CAPABLE) &&
             drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV,
-                             intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd) ==
-                             sizeof(intel_dp->edp_dpcd)))
+                             intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) ==
+                             sizeof(intel_dp->edp_dpcd))
                 DRM_DEBUG_KMS("EDP DPCD : %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
                               intel_dp->edp_dpcd);
 
@@ -3986,6 +3994,31 @@ go_again:
 }
 
 static void
+intel_dp_retrain_link(struct intel_dp *intel_dp)
+{
+        struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+
+        /* Suppress underruns caused by re-training */
+        intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
+        if (crtc->config->has_pch_encoder)
+                intel_set_pch_fifo_underrun_reporting(dev_priv,
+                                                      intel_crtc_pch_transcoder(crtc), false);
+
+        intel_dp_start_link_train(intel_dp);
+        intel_dp_stop_link_train(intel_dp);
+
+        /* Keep underrun reporting disabled until things are stable */
+        intel_wait_for_vblank(&dev_priv->drm, crtc->pipe);
+
+        intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
+        if (crtc->config->has_pch_encoder)
+                intel_set_pch_fifo_underrun_reporting(dev_priv,
+                                                      intel_crtc_pch_transcoder(crtc), true);
+}
+
+static void
 intel_dp_check_link_status(struct intel_dp *intel_dp)
 {
         struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
@@ -4005,13 +4038,18 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
         if (!to_intel_crtc(intel_encoder->base.crtc)->active)
                 return;
 
+        /* FIXME: we need to synchronize this sort of stuff with hardware
+         * readout */
+        if (WARN_ON_ONCE(!intel_dp->lane_count))
+                return;
+
         /* if link training is requested we should perform it always */
         if ((intel_dp->compliance_test_type == DP_TEST_LINK_TRAINING) ||
             (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count))) {
                 DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n",
                               intel_encoder->base.name);
-                intel_dp_start_link_train(intel_dp);
-                intel_dp_stop_link_train(intel_dp);
+
+                intel_dp_retrain_link(intel_dp);
         }
 }
 
@@ -4334,7 +4372,7 @@ intel_dp_unset_edid(struct intel_dp *intel_dp)
         intel_dp->has_audio = false;
 }
 
-static void
+static enum drm_connector_status
 intel_dp_long_pulse(struct intel_connector *intel_connector)
 {
         struct drm_connector *connector = &intel_connector->base;
@@ -4358,7 +4396,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
         else
                 status = connector_status_disconnected;
 
-        if (status != connector_status_connected) {
+        if (status == connector_status_disconnected) {
                 intel_dp->compliance_test_active = 0;
                 intel_dp->compliance_test_type = 0;
                 intel_dp->compliance_test_data = 0;
@@ -4420,8 +4458,8 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
         intel_dp->aux.i2c_defer_count = 0;
 
         intel_dp_set_edid(intel_dp);
-
-        status = connector_status_connected;
+        if (is_edp(intel_dp) || intel_connector->detect_edid)
+                status = connector_status_connected;
         intel_dp->detect_done = true;
 
         /* Try to read the source of the interrupt */
@@ -4440,12 +4478,11 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
         }
 
 out:
-        if ((status != connector_status_connected) &&
-            (intel_dp->is_mst == false))
+        if (status != connector_status_connected && !intel_dp->is_mst)
                 intel_dp_unset_edid(intel_dp);
 
         intel_display_power_put(to_i915(dev), power_domain);
-        return;
+        return status;
 }
 
 static enum drm_connector_status
@@ -4454,7 +4491,7 @@ intel_dp_detect(struct drm_connector *connector, bool force)
         struct intel_dp *intel_dp = intel_attached_dp(connector);
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        struct intel_connector *intel_connector = to_intel_connector(connector);
+        enum drm_connector_status status = connector->status;
 
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
                       connector->base.id, connector->name);
@@ -4469,14 +4506,11 @@ intel_dp_detect(struct drm_connector *connector, bool force)
 
         /* If full detect is not performed yet, do a full detect */
         if (!intel_dp->detect_done)
-                intel_dp_long_pulse(intel_dp->attached_connector);
+                status = intel_dp_long_pulse(intel_dp->attached_connector);
 
         intel_dp->detect_done = false;
 
-        if (is_edp(intel_dp) || intel_connector->detect_edid)
-                return connector_status_connected;
-        else
-                return connector_status_disconnected;
+        return status;
 }
 
 static void
@@ -4757,11 +4791,16 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp)
 void intel_dp_encoder_reset(struct drm_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->dev);
-        struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+        struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
+        struct intel_lspcon *lspcon = &intel_dig_port->lspcon;
+        struct intel_dp *intel_dp = &intel_dig_port->dp;
 
         if (!HAS_DDI(dev_priv))
                 intel_dp->DP = I915_READ(intel_dp->output_reg);
 
+        if (IS_GEN9(dev_priv) && lspcon->active)
+                lspcon_resume(lspcon);
+
         if (to_intel_encoder(encoder)->type != INTEL_OUTPUT_EDP)
                 return;
 
@@ -4828,36 +4867,34 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
                       port_name(intel_dig_port->port),
                       long_hpd ? "long" : "short");
 
+        if (long_hpd) {
+                intel_dp->detect_done = false;
+                return IRQ_NONE;
+        }
+
         power_domain = intel_display_port_aux_power_domain(intel_encoder);
         intel_display_power_get(dev_priv, power_domain);
 
-        if (long_hpd) {
-                intel_dp_long_pulse(intel_dp->attached_connector);
-                if (intel_dp->is_mst)
-                        ret = IRQ_HANDLED;
-                goto put_power;
-
-        } else {
-                if (intel_dp->is_mst) {
-                        if (intel_dp_check_mst_status(intel_dp) == -EINVAL) {
-                                /*
-                                 * If we were in MST mode, and device is not
-                                 * there, get out of MST mode
-                                 */
-                                DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n",
-                                              intel_dp->is_mst, intel_dp->mst_mgr.mst_state);
-                                intel_dp->is_mst = false;
-                                drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
-                                                                intel_dp->is_mst);
-                                goto put_power;
-                        }
+        if (intel_dp->is_mst) {
+                if (intel_dp_check_mst_status(intel_dp) == -EINVAL) {
+                        /*
+                         * If we were in MST mode, and device is not
+                         * there, get out of MST mode
+                         */
+                        DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n",
+                                      intel_dp->is_mst, intel_dp->mst_mgr.mst_state);
+                        intel_dp->is_mst = false;
+                        drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
+                                                        intel_dp->is_mst);
+                        intel_dp->detect_done = false;
+                        goto put_power;
                 }
+        }
 
-                if (!intel_dp->is_mst) {
-                        if (!intel_dp_short_pulse(intel_dp)) {
-                                intel_dp_long_pulse(intel_dp->attached_connector);
-                                goto put_power;
-                        }
+        if (!intel_dp->is_mst) {
+                if (!intel_dp_short_pulse(intel_dp)) {
+                        intel_dp->detect_done = false;
+                        goto put_power;
                 }
         }
 
@@ -5077,7 +5114,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
                  (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT);
         /* Compute the divisor for the pp clock, simply match the Bspec
          * formula. */
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 pp_div = I915_READ(regs.pp_ctrl);
                 pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK;
                 pp_div |= (DIV_ROUND_UP((seq->t11_t12 + 1), 1000)
@@ -5090,9 +5127,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
         /* Haswell doesn't have any port selection bits for the panel
          * power sequencer any more. */
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 port_sel = PANEL_PORT_SELECT_VLV(port);
-        } else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
+        } else if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
                 if (port == PORT_A)
                         port_sel = PANEL_PORT_SELECT_DPA;
                 else
@@ -5103,7 +5140,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
         I915_WRITE(regs.pp_on, pp_on);
         I915_WRITE(regs.pp_off, pp_off);
-        if (IS_BROXTON(dev))
+        if (IS_BROXTON(dev_priv))
                 I915_WRITE(regs.pp_ctrl, pp_div);
         else
                 I915_WRITE(regs.pp_div, pp_div);
@@ -5111,7 +5148,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
         DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n",
                       I915_READ(regs.pp_on),
                       I915_READ(regs.pp_off),
-                      IS_BROXTON(dev) ?
+                      IS_BROXTON(dev_priv) ?
                       (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK) :
                       I915_READ(regs.pp_div));
 }
@@ -5119,7 +5156,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 static void intel_dp_pps_init(struct drm_device *dev,
                               struct intel_dp *intel_dp)
 {
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 vlv_initial_power_sequencer_setup(intel_dp);
         } else {
                 intel_dp_init_panel_power_sequencer(dev, intel_dp);
@@ -5589,7 +5628,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         }
         mutex_unlock(&dev->mode_config.mutex);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 intel_dp->edp_notifier.notifier_call = edp_notify_handler;
                 register_reboot_notifier(&intel_dp->edp_notifier);
 
@@ -5598,7 +5637,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
                  * If the current pipe isn't valid, try the PPS pipe, and if that
                  * fails just assume pipe A.
                  */
-                if (IS_CHERRYVIEW(dev))
+                if (IS_CHERRYVIEW(dev_priv))
                         pipe = DP_PORT_TO_PIPE_CHV(intel_dp->DP);
                 else
                         pipe = PORT_TO_PIPE(intel_dp->DP);
@@ -5654,9 +5693,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         /* intel_dp vfuncs */
         if (INTEL_INFO(dev)->gen >= 9)
                 intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider;
-        else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
-        else if (HAS_PCH_SPLIT(dev))
+        else if (HAS_PCH_SPLIT(dev_priv))
                 intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider;
         else
                 intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider;
@@ -5666,7 +5705,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         else
                 intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl;
 
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev_priv))
                 intel_dp->prepare_link_retrain = intel_ddi_prepare_link_retrain;
 
         /* Preserve the current hw state. */
@@ -5687,7 +5726,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 intel_encoder->type = INTEL_OUTPUT_EDP;
 
         /* eDP only on port B and/or C on vlv/chv */
-        if (WARN_ON((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+        if (WARN_ON((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
                     is_edp(intel_dp) && port != PORT_B && port != PORT_C))
                 return false;
 
@@ -5708,7 +5747,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
 
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev_priv))
                 intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
         else
                 intel_connector->get_hw_state = intel_connector_get_hw_state;
@@ -5720,7 +5759,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 break;
         case PORT_B:
                 intel_encoder->hpd_pin = HPD_PORT_B;
-                if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+                if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                         intel_encoder->hpd_pin = HPD_PORT_A;
                 break;
         case PORT_C:
@@ -5754,7 +5793,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
          * 0xd.  Failure to do so will result in spurious interrupts being
          * generated on the port when a cable is not attached.
          */
-        if (IS_G4X(dev) && !IS_GM45(dev)) {
+        if (IS_G4X(dev_priv) && !IS_GM45(dev_priv)) {
                 u32 temp = I915_READ(PEG_BAND_GAP_DATA);
                 I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
         }
@@ -5797,13 +5836,13 @@ bool intel_dp_init(struct drm_device *dev,
         intel_encoder->get_hw_state = intel_dp_get_hw_state;
         intel_encoder->get_config = intel_dp_get_config;
         intel_encoder->suspend = intel_dp_encoder_suspend;
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(dev_priv)) {
                 intel_encoder->pre_pll_enable = chv_dp_pre_pll_enable;
                 intel_encoder->pre_enable = chv_pre_enable_dp;
                 intel_encoder->enable = vlv_enable_dp;
                 intel_encoder->post_disable = chv_post_disable_dp;
                 intel_encoder->post_pll_disable = chv_dp_post_pll_disable;
-        } else if (IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv)) {
                 intel_encoder->pre_pll_enable = vlv_dp_pre_pll_enable;
                 intel_encoder->pre_enable = vlv_pre_enable_dp;
                 intel_encoder->enable = vlv_enable_dp;
@@ -5820,7 +5859,7 @@ bool intel_dp_init(struct drm_device *dev,
         intel_dig_port->max_lanes = 4;
 
         intel_encoder->type = INTEL_OUTPUT_DP;
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(dev_priv)) {
                 if (port == PORT_D)
                         intel_encoder->crtc_mask = 1 << 2;
                 else
@@ -5829,6 +5868,7 @@ bool intel_dp_init(struct drm_device *dev,
                 intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
         }
         intel_encoder->cloneable = 0;
+        intel_encoder->port = port;
 
         intel_dig_port->hpd_pulse = intel_dp_hpd_pulse;
         dev_priv->hotplug.irq_port[port] = intel_dig_port;
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index c438b02184cb..0048b520baf7 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -225,9 +225,6 @@ static u32 intel_dp_training_pattern(struct intel_dp *intel_dp)
          * Intel platforms that support HBR2 also support TPS3. TPS3 support is
          * also mandatory for downstream devices that support HBR2. However, not
          * all sinks follow the spec.
-         *
-         * Due to WaDisableHBR2 SKL < B0 is the only exception where TPS3 is
-         * supported in source but still not enabled.
          */
         source_tps3 = intel_dp_source_supports_hbr2(intel_dp);
         sink_tps3 = drm_dp_tps3_supported(intel_dp->dpcd);
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 54a9d7610d8f..3ffbd69e4551 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -523,6 +523,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum
                          DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe));
 
         intel_encoder->type = INTEL_OUTPUT_DP_MST;
+        intel_encoder->port = intel_dig_port->port;
         intel_encoder->crtc_mask = 0x7;
         intel_encoder->cloneable = 0;
 
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index c26d18a574b6..605d0b509f24 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -1694,21 +1694,32 @@ bool bxt_ddi_dp_set_dpll_hw_state(int clock,
         return bxt_ddi_set_dpll_hw_state(clock, &clk_div, dpll_hw_state);
 }
 
+static bool
+bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc *intel_crtc,
+                               struct intel_crtc_state *crtc_state, int clock,
+                               struct intel_dpll_hw_state *dpll_hw_state)
+{
+        struct bxt_clk_div clk_div = { };
+
+        bxt_ddi_hdmi_pll_dividers(intel_crtc, crtc_state, clock, &clk_div);
+
+        return bxt_ddi_set_dpll_hw_state(clock, &clk_div, dpll_hw_state);
+}
+
 static struct intel_shared_dpll *
 bxt_get_dpll(struct intel_crtc *crtc,
                 struct intel_crtc_state *crtc_state,
                 struct intel_encoder *encoder)
 {
-        struct bxt_clk_div clk_div = {0};
-        struct intel_dpll_hw_state dpll_hw_state = {0};
+        struct intel_dpll_hw_state dpll_hw_state = { };
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_digital_port *intel_dig_port;
         struct intel_shared_dpll *pll;
         int i, clock = crtc_state->port_clock;
 
-        if (encoder->type == INTEL_OUTPUT_HDMI
-            && !bxt_ddi_hdmi_pll_dividers(crtc, crtc_state,
-                                          clock, &clk_div))
+        if (encoder->type == INTEL_OUTPUT_HDMI &&
+            !bxt_ddi_hdmi_set_dpll_hw_state(crtc, crtc_state, clock,
+                                            &dpll_hw_state))
                 return NULL;
 
         if ((encoder->type == INTEL_OUTPUT_DP ||
@@ -1840,13 +1851,13 @@ void intel_shared_dpll_init(struct drm_device *dev)
         const struct dpll_info *dpll_info;
         int i;
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
                 dpll_mgr = &skl_pll_mgr;
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 dpll_mgr = &bxt_pll_mgr;
-        else if (HAS_DDI(dev))
+        else if (HAS_DDI(dev_priv))
                 dpll_mgr = &hsw_pll_mgr;
-        else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+        else if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
                 dpll_mgr = &pch_pll_mgr;
 
         if (!dpll_mgr) {
@@ -1872,7 +1883,7 @@ void intel_shared_dpll_init(struct drm_device *dev)
         BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS);
 
         /* FIXME: Move this to a more suitable place */
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev_priv))
                 intel_ddi_pll_init(dev);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 5145ff264c8e..4e90b0760f3f 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -206,6 +206,7 @@ struct intel_encoder {
         struct drm_encoder base;
 
         enum intel_output_type type;
+        enum port port;
         unsigned int cloneable;
         void (*hot_plug)(struct intel_encoder *);
         bool (*compute_config)(struct intel_encoder *,
@@ -247,6 +248,8 @@ struct intel_encoder {
         void (*suspend)(struct intel_encoder *);
         int crtc_mask;
         enum hpd_pin hpd_pin;
+        /* for communication with audio component; protected by av_mutex */
+        const struct drm_connector *audio_connector;
 };
 
 struct intel_panel {
@@ -263,6 +266,7 @@ struct intel_panel {
                 bool enabled;
                 bool combination_mode;  /* gen 2/4 only */
                 bool active_low_pwm;
+                bool alternate_pwm_increment;   /* lpt+ */
 
                 /* PWM chip */
                 bool util_pin_active_low;       /* bxt+ */
@@ -464,9 +468,13 @@ struct intel_pipe_wm {
         bool sprites_scaled;
 };
 
-struct skl_pipe_wm {
+struct skl_plane_wm {
         struct skl_wm_level wm[8];
         struct skl_wm_level trans_wm;
+};
+
+struct skl_pipe_wm {
+        struct skl_plane_wm planes[I915_MAX_PLANES];
         uint32_t linetime;
 };
 
@@ -492,6 +500,7 @@ struct intel_crtc_wm_state {
                 struct {
                         /* gen9+ only needs 1-step wm programming */
                         struct skl_pipe_wm optimal;
+                        struct skl_ddb_entry ddb;
 
                         /* cached plane data rate */
                         unsigned plane_data_rate[I915_MAX_PLANES];
@@ -729,6 +738,9 @@ struct intel_crtc {
                 bool cxsr_allowed;
         } wm;
 
+        /* gen9+: ddb allocation currently being used */
+        struct skl_ddb_entry hw_ddb;
+
         int scanline_offset;
 
         struct {
@@ -795,22 +807,22 @@ struct intel_plane {
 };
 
 struct intel_watermark_params {
-        unsigned long fifo_size;
-        unsigned long max_wm;
-        unsigned long default_wm;
-        unsigned long guard_size;
-        unsigned long cacheline_size;
+        u16 fifo_size;
+        u16 max_wm;
+        u8 default_wm;
+        u8 guard_size;
+        u8 cacheline_size;
 };
 
 struct cxsr_latency {
-        int is_desktop;
-        int is_ddr3;
-        unsigned long fsb_freq;
-        unsigned long mem_freq;
-        unsigned long display_sr;
-        unsigned long display_hpll_disable;
-        unsigned long cursor_sr;
-        unsigned long cursor_hpll_disable;
+        bool is_desktop : 1;
+        bool is_ddr3 : 1;
+        u16 fsb_freq;
+        u16 mem_freq;
+        u16 display_sr;
+        u16 display_hpll_disable;
+        u16 cursor_sr;
+        u16 cursor_hpll_disable;
 };
 
 #define to_intel_atomic_state(x) container_of(x, struct intel_atomic_state, base)
@@ -949,17 +961,22 @@ struct intel_dp {
         bool compliance_test_active;
 };
 
+struct intel_lspcon {
+        bool active;
+        enum drm_lspcon_mode mode;
+        struct drm_dp_aux *aux;
+};
+
 struct intel_digital_port {
         struct intel_encoder base;
         enum port port;
         u32 saved_port_bits;
         struct intel_dp dp;
         struct intel_hdmi hdmi;
+        struct intel_lspcon lspcon;
         enum irqreturn (*hpd_pulse)(struct intel_digital_port *, bool);
         bool release_cl2_override;
         uint8_t max_lanes;
-        /* for communication with audio component; protected by av_mutex */
-        const struct drm_connector *audio_connector;
 };
 
 struct intel_dp_mst_encoder {
@@ -1181,6 +1198,7 @@ void i915_audio_component_init(struct drm_i915_private *dev_priv);
 void i915_audio_component_cleanup(struct drm_i915_private *dev_priv);
 
 /* intel_display.c */
+enum transcoder intel_crtc_pch_transcoder(struct intel_crtc *crtc);
 void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco);
 void intel_update_rawclk(struct drm_i915_private *dev_priv);
 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
@@ -1477,6 +1495,10 @@ static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bo
 {
 }
 
+static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
+{
+}
+
 static inline void intel_fbdev_restore_mode(struct drm_device *dev)
 {
 }
@@ -1503,6 +1525,7 @@ void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
 void intel_fbc_flush(struct drm_i915_private *dev_priv,
                      unsigned int frontbuffer_bits, enum fb_op_origin origin);
 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv);
+void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv);
 
 /* intel_hdmi.c */
 void intel_hdmi_init(struct drm_device *dev, i915_reg_t hdmi_reg, enum port port);
@@ -1706,7 +1729,7 @@ bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
 /* intel_pm.c */
 void intel_init_clock_gating(struct drm_device *dev);
 void intel_suspend_hw(struct drm_device *dev);
-int ilk_wm_max_level(const struct drm_device *dev);
+int ilk_wm_max_level(const struct drm_i915_private *dev_priv);
 void intel_update_watermarks(struct drm_crtc *crtc);
 void intel_init_pm(struct drm_device *dev);
 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
@@ -1732,20 +1755,24 @@ void ilk_wm_get_hw_state(struct drm_device *dev);
 void skl_wm_get_hw_state(struct drm_device *dev);
 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
                           struct skl_ddb_allocation *ddb /* out */);
-bool skl_can_enable_sagv(struct drm_atomic_state *state);
-int skl_enable_sagv(struct drm_i915_private *dev_priv);
-int skl_disable_sagv(struct drm_i915_private *dev_priv);
+void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
+                              struct skl_pipe_wm *out);
+bool intel_can_enable_sagv(struct drm_atomic_state *state);
+int intel_enable_sagv(struct drm_i915_private *dev_priv);
+int intel_disable_sagv(struct drm_i915_private *dev_priv);
+bool skl_wm_level_equals(const struct skl_wm_level *l1,
+                         const struct skl_wm_level *l2);
 bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old,
                                const struct skl_ddb_allocation *new,
                                enum pipe pipe);
 bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state,
-                                 const struct skl_ddb_allocation *old,
-                                 const struct skl_ddb_allocation *new,
-                                 enum pipe pipe);
+                                 struct intel_crtc *intel_crtc);
 void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
-                         const struct skl_wm_values *wm);
+                         const struct skl_plane_wm *wm,
+                         const struct skl_ddb_allocation *ddb);
 void skl_write_plane_wm(struct intel_crtc *intel_crtc,
-                        const struct skl_wm_values *wm,
+                        const struct skl_plane_wm *wm,
+                        const struct skl_ddb_allocation *ddb,
                         int plane);
 uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config);
 bool ilk_disable_lp_wm(struct drm_device *dev);
@@ -1825,4 +1852,7 @@ int intel_color_check(struct drm_crtc *crtc, struct drm_crtc_state *state);
 void intel_color_set_csc(struct drm_crtc_state *crtc_state);
 void intel_color_load_luts(struct drm_crtc_state *crtc_state);
 
+/* intel_lspcon.c */
+bool lspcon_init(struct intel_digital_port *intel_dig_port);
+void lspcon_resume(struct intel_lspcon *lspcon);
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index b2e3d3a334f7..4e0d025490a3 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -437,11 +437,11 @@ static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 
 static void intel_dsi_device_ready(struct intel_encoder *encoder)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 vlv_dsi_device_ready(encoder);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 bxt_dsi_device_ready(encoder);
 }
 
@@ -464,7 +464,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder)
         }
 
         for_each_dsi_port(port, intel_dsi->ports) {
-                i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+                i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
                         BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
                 u32 temp;
 
@@ -494,7 +494,7 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
         enum port port;
 
         for_each_dsi_port(port, intel_dsi->ports) {
-                i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+                i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
                         BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
                 u32 temp;
 
@@ -656,7 +656,6 @@ static void intel_dsi_disable(struct intel_encoder *encoder)
 
 static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 {
-        struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
@@ -664,7 +663,7 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
         DRM_DEBUG_KMS("\n");
         for_each_dsi_port(port, intel_dsi->ports) {
                 /* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */
-                i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+                i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
                         BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(PORT_A);
                 u32 val;
 
@@ -741,7 +740,6 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
         enum intel_display_power_domain power_domain;
         enum port port;
         bool active = false;
@@ -762,7 +760,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 
         /* XXX: this only works for one DSI output */
         for_each_dsi_port(port, intel_dsi->ports) {
-                i915_reg_t ctrl_reg = IS_BROXTON(dev) ?
+                i915_reg_t ctrl_reg = IS_BROXTON(dev_priv) ?
                         BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
                 bool enabled = I915_READ(ctrl_reg) & DPI_ENABLE;
 
@@ -771,7 +769,8 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
                  * bit in port C control register does not get set. As a
                  * workaround, check pipe B conf instead.
                  */
-                if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) && port == PORT_C)
+                if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
+                    port == PORT_C)
                         enabled = I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 
                 /* Try command mode if video mode not enabled */
@@ -970,11 +969,11 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 static void intel_dsi_get_config(struct intel_encoder *encoder,
                                  struct intel_crtc_state *pipe_config)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         u32 pclk;
         DRM_DEBUG_KMS("\n");
 
-        if (IS_BROXTON(dev))
+        if (IS_BROXTON(dev_priv))
                 bxt_dsi_get_pipe_config(encoder, pipe_config);
 
         pclk = intel_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
@@ -1066,7 +1065,7 @@ static void set_dsi_timings(struct drm_encoder *encoder,
         hbp = txbyteclkhs(hbp, bpp, lane_count, intel_dsi->burst_mode_ratio);
 
         for_each_dsi_port(port, intel_dsi->ports) {
-                if (IS_BROXTON(dev)) {
+                if (IS_BROXTON(dev_priv)) {
                         /*
                          * Program hdisplay and vdisplay on MIPI transcoder.
                          * This is different from calculated hactive and
@@ -1138,7 +1137,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
         }
 
         for_each_dsi_port(port, intel_dsi->ports) {
-                if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+                if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                         /*
                          * escape clock divider, 20MHz, shared for A and C.
                          * device ready must be off when doing this! txclkesc?
@@ -1153,7 +1152,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
                         tmp &= ~READ_REQUEST_PRIORITY_MASK;
                         I915_WRITE(MIPI_CTRL(port), tmp |
                                         READ_REQUEST_PRIORITY_HIGH);
-                } else if (IS_BROXTON(dev)) {
+                } else if (IS_BROXTON(dev_priv)) {
                         enum pipe pipe = intel_crtc->pipe;
 
                         tmp = I915_READ(MIPI_CTRL(port));
@@ -1242,7 +1241,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
                 I915_WRITE(MIPI_INIT_COUNT(port),
                                 txclkesc(intel_dsi->escape_clk_div, 100));
 
-                if (IS_BROXTON(dev) && (!intel_dsi->dual_link)) {
+                if (IS_BROXTON(dev_priv) && (!intel_dsi->dual_link)) {
                         /*
                          * BXT spec says write MIPI_INIT_COUNT for
                          * both the ports, even if only one is
@@ -1346,7 +1345,7 @@ static int intel_dsi_set_property(struct drm_connector *connector,
                         DRM_DEBUG_KMS("no scaling not supported\n");
                         return -EINVAL;
                 }
-                if (HAS_GMCH_DISPLAY(dev) &&
+                if (HAS_GMCH_DISPLAY(to_i915(dev)) &&
                     val == DRM_MODE_SCALE_CENTER) {
                         DRM_DEBUG_KMS("centering not supported\n");
                         return -EINVAL;
@@ -1450,9 +1449,9 @@ void intel_dsi_init(struct drm_device *dev)
         if (!intel_bios_is_dsi_present(dev_priv, &port))
                 return;
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
-        } else if (IS_BROXTON(dev)) {
+        } else if (IS_BROXTON(dev_priv)) {
                 dev_priv->mipi_mmio_base = BXT_MIPI_BASE;
         } else {
                 DRM_ERROR("Unsupported Mipi device to reg base");
@@ -1488,6 +1487,7 @@ void intel_dsi_init(struct drm_device *dev)
 
         intel_connector->get_hw_state = intel_connector_get_hw_state;
 
+        intel_encoder->port = port;
         /*
          * On BYT/CHV, pipe A maps to MIPI DSI port A, pipe B maps to MIPI DSI
          * port C. BXT isn't limited like this.
diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
index cd154ce6b6c1..9f279a3d0f74 100644
--- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
@@ -126,6 +126,8 @@ static const u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi,
         u16 len;
         enum port port;
 
+        DRM_DEBUG_KMS("\n");
+
         flags = *data++;
         type = *data++;
 
@@ -199,6 +201,8 @@ static const u8 *mipi_exec_delay(struct intel_dsi *intel_dsi, const u8 *data)
 {
         u32 delay = *((const u32 *) data);
 
+        DRM_DEBUG_KMS("\n");
+
         usleep_range(delay, delay + 10);
         data += 4;
 
@@ -307,6 +311,8 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data)
         u8 gpio_source, gpio_index;
         bool value;
 
+        DRM_DEBUG_KMS("\n");
+
         if (dev_priv->vbt.dsi.seq_version >= 3)
                 data++;
 
@@ -331,18 +337,36 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data)
         return data;
 }
 
-static const u8 *mipi_exec_i2c_skip(struct intel_dsi *intel_dsi, const u8 *data)
+static const u8 *mipi_exec_i2c(struct intel_dsi *intel_dsi, const u8 *data)
 {
+        DRM_DEBUG_KMS("Skipping I2C element execution\n");
+
         return data + *(data + 6) + 7;
 }
 
+static const u8 *mipi_exec_spi(struct intel_dsi *intel_dsi, const u8 *data)
+{
+        DRM_DEBUG_KMS("Skipping SPI element execution\n");
+
+        return data + *(data + 5) + 6;
+}
+
+static const u8 *mipi_exec_pmic(struct intel_dsi *intel_dsi, const u8 *data)
+{
+        DRM_DEBUG_KMS("Skipping PMIC element execution\n");
+
+        return data + 15;
+}
+
 typedef const u8 * (*fn_mipi_elem_exec)(struct intel_dsi *intel_dsi,
                                         const u8 *data);
 static const fn_mipi_elem_exec exec_elem[] = {
         [MIPI_SEQ_ELEM_SEND_PKT] = mipi_exec_send_packet,
         [MIPI_SEQ_ELEM_DELAY] = mipi_exec_delay,
         [MIPI_SEQ_ELEM_GPIO] = mipi_exec_gpio,
-        [MIPI_SEQ_ELEM_I2C] = mipi_exec_i2c_skip,
+        [MIPI_SEQ_ELEM_I2C] = mipi_exec_i2c,
+        [MIPI_SEQ_ELEM_SPI] = mipi_exec_spi,
+        [MIPI_SEQ_ELEM_PMIC] = mipi_exec_pmic,
 };
 
 /*
@@ -385,11 +409,8 @@ static void generic_exec_sequence(struct drm_panel *panel, enum mipi_seq seq_id)
                 return;
 
         data = dev_priv->vbt.dsi.sequence[seq_id];
-        if (!data) {
-                DRM_DEBUG_KMS("MIPI sequence %d - %s not available\n",
-                              seq_id, sequence_name(seq_id));
+        if (!data)
                 return;
-        }
 
         WARN_ON(*data != seq_id);
 
@@ -420,7 +441,15 @@ static void generic_exec_sequence(struct drm_panel *panel, enum mipi_seq seq_id)
                         operation_size = *data++;
 
                 if (mipi_elem_exec) {
+                        const u8 *next = data + operation_size;
+
                         data = mipi_elem_exec(intel_dsi, data);
+
+                        /* Consistency check if we have size. */
+                        if (operation_size && data != next) {
+                                DRM_ERROR("Inconsistent operation size\n");
+                                return;
+                        }
                 } else if (operation_size) {
                         /* We have size, skip. */
                         DRM_DEBUG_KMS("Unsupported MIPI operation byte %u\n",
@@ -438,6 +467,8 @@ static void generic_exec_sequence(struct drm_panel *panel, enum mipi_seq seq_id)
 static int vbt_panel_prepare(struct drm_panel *panel)
 {
         generic_exec_sequence(panel, MIPI_SEQ_ASSERT_RESET);
+        generic_exec_sequence(panel, MIPI_SEQ_POWER_ON);
+        generic_exec_sequence(panel, MIPI_SEQ_DEASSERT_RESET);
         generic_exec_sequence(panel, MIPI_SEQ_INIT_OTP);
 
         return 0;
@@ -445,7 +476,8 @@ static int vbt_panel_prepare(struct drm_panel *panel)
 
 static int vbt_panel_unprepare(struct drm_panel *panel)
 {
-        generic_exec_sequence(panel, MIPI_SEQ_DEASSERT_RESET);
+        generic_exec_sequence(panel, MIPI_SEQ_ASSERT_RESET);
+        generic_exec_sequence(panel, MIPI_SEQ_POWER_OFF);
 
         return 0;
 }
@@ -453,12 +485,14 @@ static int vbt_panel_unprepare(struct drm_panel *panel)
 static int vbt_panel_enable(struct drm_panel *panel)
 {
         generic_exec_sequence(panel, MIPI_SEQ_DISPLAY_ON);
+        generic_exec_sequence(panel, MIPI_SEQ_BACKLIGHT_ON);
 
         return 0;
 }
 
 static int vbt_panel_disable(struct drm_panel *panel)
 {
+        generic_exec_sequence(panel, MIPI_SEQ_BACKLIGHT_OFF);
         generic_exec_sequence(panel, MIPI_SEQ_DISPLAY_OFF);
 
         return 0;
diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c
index 6ab58a01b18e..56eff6004bc0 100644
--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
+++ b/drivers/gpu/drm/i915/intel_dsi_pll.c
@@ -351,7 +351,7 @@ static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
 u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
                        struct intel_crtc_state *config)
 {
-        if (IS_BROXTON(encoder->base.dev))
+        if (IS_BROXTON(to_i915(encoder->base.dev)))
                 return bxt_dsi_get_pclk(encoder, pipe_bpp, config);
         else
                 return vlv_dsi_get_pclk(encoder, pipe_bpp, config);
@@ -515,11 +515,11 @@ bool intel_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
 int intel_compute_dsi_pll(struct intel_encoder *encoder,
                           struct intel_crtc_state *config)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 return vlv_compute_dsi_pll(encoder, config);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 return bxt_compute_dsi_pll(encoder, config);
 
         return -ENODEV;
@@ -528,21 +528,21 @@ int intel_compute_dsi_pll(struct intel_encoder *encoder,
 void intel_enable_dsi_pll(struct intel_encoder *encoder,
                           const struct intel_crtc_state *config)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 vlv_enable_dsi_pll(encoder, config);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 bxt_enable_dsi_pll(encoder, config);
 }
 
 void intel_disable_dsi_pll(struct intel_encoder *encoder)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 vlv_disable_dsi_pll(encoder);
-        else if (IS_BROXTON(dev))
+        else if (IS_BROXTON(dev_priv))
                 bxt_disable_dsi_pll(encoder);
 }
 
@@ -564,10 +564,10 @@ static void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 
 void intel_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (IS_BROXTON(dev))
+        if (IS_BROXTON(dev_priv))
                 bxt_dsi_reset_clocks(encoder, port);
-        else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 vlv_dsi_reset_clocks(encoder, port);
 }
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 2e452c505e7e..cd574900cd8d 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -412,16 +412,14 @@ intel_dvo_get_current_mode(struct drm_connector *connector)
         return mode;
 }
 
-static char intel_dvo_port_name(i915_reg_t dvo_reg)
+static enum port intel_dvo_port(i915_reg_t dvo_reg)
 {
         if (i915_mmio_reg_equal(dvo_reg, DVOA))
-                return 'A';
+                return PORT_A;
         else if (i915_mmio_reg_equal(dvo_reg, DVOB))
-                return 'B';
-        else if (i915_mmio_reg_equal(dvo_reg, DVOC))
-                return 'C';
+                return PORT_B;
         else
-                return '?';
+                return PORT_C;
 }
 
 void intel_dvo_init(struct drm_device *dev)
@@ -464,6 +462,7 @@ void intel_dvo_init(struct drm_device *dev)
                 bool dvoinit;
                 enum pipe pipe;
                 uint32_t dpll[I915_MAX_PIPES];
+                enum port port;
 
                 /* Allow the I2C driver info to specify the GPIO to be used in
                  * special cases, but otherwise default to what's defined
@@ -511,12 +510,15 @@ void intel_dvo_init(struct drm_device *dev)
                 if (!dvoinit)
                         continue;
 
+                port = intel_dvo_port(dvo->dvo_reg);
                 drm_encoder_init(dev, &intel_encoder->base,
                                  &intel_dvo_enc_funcs, encoder_type,
-                                 "DVO %c", intel_dvo_port_name(dvo->dvo_reg));
+                                 "DVO %c", port_name(port));
 
                 intel_encoder->type = INTEL_OUTPUT_DVO;
+                intel_encoder->port = port;
                 intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
+
                 switch (dvo->type) {
                 case INTEL_DVO_CHIP_TMDS:
                         intel_encoder->cloneable = (1 << INTEL_OUTPUT_ANALOG) |
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index e405f1080296..2dc94812bea5 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -82,12 +82,17 @@ static const struct engine_info {
         },
 };
 
-static struct intel_engine_cs *
+static int
 intel_engine_setup(struct drm_i915_private *dev_priv,
                    enum intel_engine_id id)
 {
         const struct engine_info *info = &intel_engines[id];
-        struct intel_engine_cs *engine = &dev_priv->engine[id];
+        struct intel_engine_cs *engine;
+
+        GEM_BUG_ON(dev_priv->engine[id]);
+        engine = kzalloc(sizeof(*engine), GFP_KERNEL);
+        if (!engine)
+                return -ENOMEM;
 
         engine->id = id;
         engine->i915 = dev_priv;
@@ -97,7 +102,8 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
         engine->mmio_base = info->mmio_base;
         engine->irq_shift = info->irq_shift;
 
-        return engine;
+        dev_priv->engine[id] = engine;
+        return 0;
 }
 
 /**
@@ -110,13 +116,16 @@ int intel_engines_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
+        unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
         unsigned int mask = 0;
         int (*init)(struct intel_engine_cs *engine);
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         unsigned int i;
         int ret;
 
-        WARN_ON(INTEL_INFO(dev_priv)->ring_mask == 0);
-        WARN_ON(INTEL_INFO(dev_priv)->ring_mask &
+        WARN_ON(ring_mask == 0);
+        WARN_ON(ring_mask &
                 GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
 
         for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
@@ -131,7 +140,11 @@ int intel_engines_init(struct drm_device *dev)
                 if (!init)
                         continue;
 
-                ret = init(intel_engine_setup(dev_priv, i));
+                ret = intel_engine_setup(dev_priv, i);
+                if (ret)
+                        goto cleanup;
+
+                ret = init(dev_priv->engine[i]);
                 if (ret)
                         goto cleanup;
 
@@ -143,7 +156,7 @@ int intel_engines_init(struct drm_device *dev)
          * are added to the driver by a warning and disabling the forgotten
          * engines.
          */
-        if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask))
+        if (WARN_ON(mask != ring_mask))
                 device_info->ring_mask = mask;
 
         device_info->num_rings = hweight32(mask);
@@ -151,11 +164,11 @@ int intel_engines_init(struct drm_device *dev)
         return 0;
 
 cleanup:
-        for (i = 0; i < I915_NUM_ENGINES; i++) {
+        for_each_engine(engine, dev_priv, id) {
                 if (i915.enable_execlists)
-                        intel_logical_ring_cleanup(&dev_priv->engine[i]);
+                        intel_logical_ring_cleanup(engine);
                 else
-                        intel_engine_cleanup(&dev_priv->engine[i]);
+                        intel_engine_cleanup(engine);
         }
 
         return ret;
@@ -210,9 +223,6 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno)
 void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
 {
         memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
-        clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
-        if (intel_engine_has_waiter(engine))
-                i915_queue_hangcheck(engine->i915);
 }
 
 static void intel_engine_init_requests(struct intel_engine_cs *engine)
@@ -307,18 +317,6 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
         return 0;
 }
 
-void intel_engine_reset_irq(struct intel_engine_cs *engine)
-{
-        struct drm_i915_private *dev_priv = engine->i915;
-
-        spin_lock_irq(&dev_priv->irq_lock);
-        if (intel_engine_has_waiter(engine))
-                engine->irq_enable(engine);
-        else
-                engine->irq_disable(engine);
-        spin_unlock_irq(&dev_priv->irq_lock);
-}
-
 /**
  * intel_engines_cleanup_common - cleans up the engine state created by
  *                                the common initiailizers.
@@ -334,3 +332,137 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
         intel_engine_cleanup_cmd_parser(engine);
         i915_gem_batch_pool_fini(&engine->batch_pool);
 }
+
+u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
+{
+        struct drm_i915_private *dev_priv = engine->i915;
+        u64 acthd;
+
+        if (INTEL_GEN(dev_priv) >= 8)
+                acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
+                                         RING_ACTHD_UDW(engine->mmio_base));
+        else if (INTEL_GEN(dev_priv) >= 4)
+                acthd = I915_READ(RING_ACTHD(engine->mmio_base));
+        else
+                acthd = I915_READ(ACTHD);
+
+        return acthd;
+}
+
+u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
+{
+        struct drm_i915_private *dev_priv = engine->i915;
+        u64 bbaddr;
+
+        if (INTEL_GEN(dev_priv) >= 8)
+                bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
+                                          RING_BBADDR_UDW(engine->mmio_base));
+        else
+                bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
+
+        return bbaddr;
+}
+
+const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
+{
+        switch (type) {
+        case I915_CACHE_NONE: return " uncached";
+        case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
+        case I915_CACHE_L3_LLC: return " L3+LLC";
+        case I915_CACHE_WT: return " WT";
+        default: return "";
+        }
+}
+
+static inline uint32_t
+read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
+                  int subslice, i915_reg_t reg)
+{
+        uint32_t mcr;
+        uint32_t ret;
+        enum forcewake_domains fw_domains;
+
+        fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
+                                                    FW_REG_READ);
+        fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+                                                     GEN8_MCR_SELECTOR,
+                                                     FW_REG_READ | FW_REG_WRITE);
+
+        spin_lock_irq(&dev_priv->uncore.lock);
+        intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
+
+        mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
+        /*
+         * The HW expects the slice and sublice selectors to be reset to 0
+         * after reading out the registers.
+         */
+        WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
+        mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
+        mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
+        I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+
+        ret = I915_READ_FW(reg);
+
+        mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
+        I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+
+        intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
+        spin_unlock_irq(&dev_priv->uncore.lock);
+
+        return ret;
+}
+
+/* NB: please notice the memset */
+void intel_engine_get_instdone(struct intel_engine_cs *engine,
+                               struct intel_instdone *instdone)
+{
+        struct drm_i915_private *dev_priv = engine->i915;
+        u32 mmio_base = engine->mmio_base;
+        int slice;
+        int subslice;
+
+        memset(instdone, 0, sizeof(*instdone));
+
+        switch (INTEL_GEN(dev_priv)) {
+        default:
+                instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+
+                if (engine->id != RCS)
+                        break;
+
+                instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
+                for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
+                        instdone->sampler[slice][subslice] =
+                                read_subslice_reg(dev_priv, slice, subslice,
+                                                  GEN7_SAMPLER_INSTDONE);
+                        instdone->row[slice][subslice] =
+                                read_subslice_reg(dev_priv, slice, subslice,
+                                                  GEN7_ROW_INSTDONE);
+                }
+                break;
+        case 7:
+                instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+
+                if (engine->id != RCS)
+                        break;
+
+                instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
+                instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
+                instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
+
+                break;
+        case 6:
+        case 5:
+        case 4:
+                instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+
+                if (engine->id == RCS)
+                        /* HACK: Using the wrong struct member */
+                        instdone->slice_common = I915_READ(GEN4_INSTDONE1);
+                break;
+        case 3:
+        case 2:
+                instdone->instdone = I915_READ(GEN2_INSTDONE);
+                break;
+        }
+}
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index afc040be1172..49048d9d7895 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -774,6 +774,14 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct intel_fbc_state_cache *cache = &fbc->state_cache;
 
+        /* We don't need to use a state cache here since this information is
+         * global for all CRTC.
+         */
+        if (fbc->underrun_detected) {
+                fbc->no_fbc_reason = "underrun detected";
+                return false;
+        }
+
         if (!cache->plane.visible) {
                 fbc->no_fbc_reason = "primary plane not visible";
                 return false;
@@ -859,6 +867,11 @@ static bool intel_fbc_can_choose(struct intel_crtc *crtc)
                 return false;
         }
 
+        if (fbc->underrun_detected) {
+                fbc->no_fbc_reason = "underrun detected";
+                return false;
+        }
+
         if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A) {
                 fbc->no_fbc_reason = "no enabled pipes can have FBC";
                 return false;
@@ -1221,6 +1234,59 @@ void intel_fbc_global_disable(struct drm_i915_private *dev_priv)
         cancel_work_sync(&fbc->work.work);
 }
 
+static void intel_fbc_underrun_work_fn(struct work_struct *work)
+{
+        struct drm_i915_private *dev_priv =
+                container_of(work, struct drm_i915_private, fbc.underrun_work);
+        struct intel_fbc *fbc = &dev_priv->fbc;
+
+        mutex_lock(&fbc->lock);
+
+        /* Maybe we were scheduled twice. */
+        if (fbc->underrun_detected)
+                goto out;
+
+        DRM_DEBUG_KMS("Disabling FBC due to FIFO underrun.\n");
+        fbc->underrun_detected = true;
+
+        intel_fbc_deactivate(dev_priv);
+out:
+        mutex_unlock(&fbc->lock);
+}
+
+/**
+ * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
+ * @dev_priv: i915 device instance
+ *
+ * Without FBC, most underruns are harmless and don't really cause too many
+ * problems, except for an annoying message on dmesg. With FBC, underruns can
+ * become black screens or even worse, especially when paired with bad
+ * watermarks. So in order for us to be on the safe side, completely disable FBC
+ * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
+ * already suggests that watermarks may be bad, so try to be as safe as
+ * possible.
+ *
+ * This function is called from the IRQ handler.
+ */
+void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv)
+{
+        struct intel_fbc *fbc = &dev_priv->fbc;
+
+        if (!fbc_supported(dev_priv))
+                return;
+
+        /* There's no guarantee that underrun_detected won't be set to true
+         * right after this check and before the work is scheduled, but that's
+         * not a problem since we'll check it again under the work function
+         * while FBC is locked. This check here is just to prevent us from
+         * unnecessarily scheduling the work, and it relies on the fact that we
+         * never switch underrun_detect back to false after it's true. */
+        if (READ_ONCE(fbc->underrun_detected))
+                return;
+
+        schedule_work(&fbc->underrun_work);
+}
+
 /**
  * intel_fbc_init_pipe_state - initialize FBC's CRTC visibility tracking
  * @dev_priv: i915 device instance
@@ -1292,6 +1358,7 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
         enum pipe pipe;
 
         INIT_WORK(&fbc->work.work, intel_fbc_work_fn);
+        INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
         mutex_init(&fbc->lock);
         fbc->enabled = false;
         fbc->active = false;
diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c
index 2aa744081f09..3018f4f589c8 100644
--- a/drivers/gpu/drm/i915/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c
@@ -254,13 +254,13 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
         old = !intel_crtc->cpu_fifo_underrun_disabled;
         intel_crtc->cpu_fifo_underrun_disabled = !enable;
 
-        if (HAS_GMCH_DISPLAY(dev))
+        if (HAS_GMCH_DISPLAY(dev_priv))
                 i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old);
-        else if (IS_GEN5(dev) || IS_GEN6(dev))
+        else if (IS_GEN5(dev_priv) || IS_GEN6(dev_priv))
                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
-        else if (IS_GEN7(dev))
+        else if (IS_GEN7(dev_priv))
                 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable, old);
-        else if (IS_GEN8(dev) || IS_GEN9(dev))
+        else if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv))
                 broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
 
         return old;
@@ -372,6 +372,8 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
         if (intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false))
                 DRM_ERROR("CPU pipe %c FIFO underrun\n",
                           pipe_name(pipe));
+
+        intel_fbc_handle_fifo_underrun_irq(dev_priv);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index b1ba86958811..5cdf7aa75be5 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -160,6 +160,7 @@ extern int intel_guc_resume(struct drm_device *dev);
 int i915_guc_submission_init(struct drm_i915_private *dev_priv);
 int i915_guc_submission_enable(struct drm_i915_private *dev_priv);
 int i915_guc_wq_reserve(struct drm_i915_gem_request *rq);
+void i915_guc_wq_unreserve(struct drm_i915_gem_request *request);
 void i915_guc_submission_disable(struct drm_i915_private *dev_priv);
 void i915_guc_submission_fini(struct drm_i915_private *dev_priv);
 
diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c
index 6fd39efb7894..3c8eaae13732 100644
--- a/drivers/gpu/drm/i915/intel_guc_loader.c
+++ b/drivers/gpu/drm/i915/intel_guc_loader.c
@@ -100,12 +100,13 @@ const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status)
 static void guc_interrupts_release(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int irqs;
 
         /* tell all command streamers NOT to forward interrupts or vblank to GuC */
         irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_NEVER);
         irqs |= _MASKED_BIT_DISABLE(GFX_INTERRUPT_STEERING);
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MODE_GEN7(engine), irqs);
 
         /* route all GT interrupts to the host */
@@ -117,12 +118,13 @@ static void guc_interrupts_release(struct drm_i915_private *dev_priv)
 static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int irqs;
         u32 tmp;
 
         /* tell all command streamers to forward interrupts (but not vblank) to GuC */
         irqs = _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MODE_GEN7(engine), irqs);
 
         /* route USER_INTERRUPT to Host, all others are sent to GuC. */
@@ -347,7 +349,6 @@ static u32 guc_wopcm_size(struct drm_i915_private *dev_priv)
 static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 {
         struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
-        struct drm_device *dev = &dev_priv->drm;
         struct i915_vma *vma;
         int ret;
 
@@ -375,24 +376,22 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
         /* Enable MIA caching. GuC clock gating is disabled. */
         I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE);
 
-        /* WaDisableMinuteIaClockGating:skl,bxt */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        /* WaDisableMinuteIaClockGating:bxt */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 I915_WRITE(GUC_SHIM_CONTROL, (I915_READ(GUC_SHIM_CONTROL) &
                                               ~GUC_ENABLE_MIA_CLOCK_GATING));
         }
 
-        /* WaC6DisallowByGfxPause*/
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_C0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_B0))
+        /* WaC6DisallowByGfxPause:bxt */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
                 I915_WRITE(GEN6_GFXPAUSE, 0x30FFF);
 
-        if (IS_BROXTON(dev))
+        if (IS_BROXTON(dev_priv))
                 I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
         else
                 I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
 
-        if (IS_GEN9(dev)) {
+        if (IS_GEN9(dev_priv)) {
                 /* DOP Clock Gating Enable for GuC clocks */
                 I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
                                             I915_READ(GEN7_MISCCPCTL)));
@@ -720,23 +719,28 @@ void intel_guc_init(struct drm_device *dev)
         struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
         const char *fw_path;
 
-        /* A negative value means "use platform default" */
-        if (i915.enable_guc_loading < 0)
-                i915.enable_guc_loading = HAS_GUC_UCODE(dev);
-        if (i915.enable_guc_submission < 0)
-                i915.enable_guc_submission = HAS_GUC_SCHED(dev);
+        if (!HAS_GUC(dev)) {
+                i915.enable_guc_loading = 0;
+                i915.enable_guc_submission = 0;
+        } else {
+                /* A negative value means "use platform default" */
+                if (i915.enable_guc_loading < 0)
+                        i915.enable_guc_loading = HAS_GUC_UCODE(dev);
+                if (i915.enable_guc_submission < 0)
+                        i915.enable_guc_submission = HAS_GUC_SCHED(dev);
+        }
 
         if (!HAS_GUC_UCODE(dev)) {
                 fw_path = NULL;
-        } else if (IS_SKYLAKE(dev)) {
+        } else if (IS_SKYLAKE(dev_priv)) {
                 fw_path = I915_SKL_GUC_UCODE;
                 guc_fw->guc_fw_major_wanted = SKL_FW_MAJOR;
                 guc_fw->guc_fw_minor_wanted = SKL_FW_MINOR;
-        } else if (IS_BROXTON(dev)) {
+        } else if (IS_BROXTON(dev_priv)) {
                 fw_path = I915_BXT_GUC_UCODE;
                 guc_fw->guc_fw_major_wanted = BXT_FW_MAJOR;
                 guc_fw->guc_fw_minor_wanted = BXT_FW_MINOR;
-        } else if (IS_KABYLAKE(dev)) {
+        } else if (IS_KABYLAKE(dev_priv)) {
                 fw_path = I915_KBL_GUC_UCODE;
                 guc_fw->guc_fw_major_wanted = KBL_FW_MAJOR;
                 guc_fw->guc_fw_minor_wanted = KBL_FW_MINOR;
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
index 434f4d5c553d..290384e86c63 100644
--- a/drivers/gpu/drm/i915/intel_gvt.c
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -31,14 +31,20 @@
  * GPU among multiple virtual machines on a time-sharing basis. Each
  * virtual machine is presented a virtual GPU (vGPU), which has equivalent
  * features as the underlying physical GPU (pGPU), so i915 driver can run
- * seamlessly in a virtual machine. This file provides the englightments
- * of GVT and the necessary components used by GVT in i915 driver.
+ * seamlessly in a virtual machine.
+ *
+ * To virtualize GPU resources GVT-g driver depends on hypervisor technology
+ * e.g KVM/VFIO/mdev, Xen, etc. to provide resource access trapping capability
+ * and be virtualized within GVT-g device module. More architectural design
+ * doc is available on https://01.org/group/2230/documentation-list.
  */
 
 static bool is_supported_device(struct drm_i915_private *dev_priv)
 {
         if (IS_BROADWELL(dev_priv))
                 return true;
+        if (IS_SKYLAKE(dev_priv))
+                return true;
         return false;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_gvt.h b/drivers/gpu/drm/i915/intel_gvt.h
index 960211df74db..25df2d65b985 100644
--- a/drivers/gpu/drm/i915/intel_gvt.h
+++ b/drivers/gpu/drm/i915/intel_gvt.h
@@ -24,7 +24,7 @@
 #ifndef _INTEL_GVT_H_
 #define _INTEL_GVT_H_
 
-#include "gvt/gvt.h"
+struct intel_gvt;
 
 #ifdef CONFIG_DRM_I915_GVT
 int intel_gvt_init(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index f40a35f2913a..af8715f31807 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -50,7 +50,7 @@ assert_hdmi_port_disabled(struct intel_hdmi *intel_hdmi)
         struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t enabled_bits;
 
-        enabled_bits = HAS_DDI(dev) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE;
+        enabled_bits = HAS_DDI(dev_priv) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE;
 
         WARN(I915_READ(intel_hdmi->hdmi_reg) & enabled_bits,
              "HDMI port enabled, expecting disabled\n");
@@ -864,7 +864,7 @@ static void intel_hdmi_prepare(struct intel_encoder *encoder)
         intel_dp_dual_mode_set_tmds_output(intel_hdmi, true);
 
         hdmi_val = SDVO_ENCODING_HDMI;
-        if (!HAS_PCH_SPLIT(dev) && crtc->config->limited_color_range)
+        if (!HAS_PCH_SPLIT(dev_priv) && crtc->config->limited_color_range)
                 hdmi_val |= HDMI_COLOR_RANGE_16_235;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                 hdmi_val |= SDVO_VSYNC_ACTIVE_HIGH;
@@ -879,9 +879,9 @@ static void intel_hdmi_prepare(struct intel_encoder *encoder)
         if (crtc->config->has_hdmi_sink)
                 hdmi_val |= HDMI_MODE_SELECT_HDMI;
 
-        if (HAS_PCH_CPT(dev))
+        if (HAS_PCH_CPT(dev_priv))
                 hdmi_val |= SDVO_PIPE_SEL_CPT(crtc->pipe);
-        else if (IS_CHERRYVIEW(dev))
+        else if (IS_CHERRYVIEW(dev_priv))
                 hdmi_val |= SDVO_PIPE_SEL_CHV(crtc->pipe);
         else
                 hdmi_val |= SDVO_PIPE_SEL(crtc->pipe);
@@ -911,9 +911,9 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder,
         if (!(tmp & SDVO_ENABLE))
                 goto out;
 
-        if (HAS_PCH_CPT(dev))
+        if (HAS_PCH_CPT(dev_priv))
                 *pipe = PORT_TO_PIPE_CPT(tmp);
-        else if (IS_CHERRYVIEW(dev))
+        else if (IS_CHERRYVIEW(dev_priv))
                 *pipe = SDVO_PORT_TO_PIPE_CHV(tmp);
         else
                 *pipe = PORT_TO_PIPE(tmp);
@@ -956,7 +956,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
         if (tmp & SDVO_AUDIO_ENABLE)
                 pipe_config->has_audio = true;
 
-        if (!HAS_PCH_SPLIT(dev) &&
+        if (!HAS_PCH_SPLIT(dev_priv) &&
             tmp & HDMI_COLOR_RANGE_16_235)
                 pipe_config->limited_color_range = true;
 
@@ -1141,7 +1141,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder,
          * to transcoder A after disabling it to allow the
          * matching DP port to be enabled on transcoder A.
          */
-        if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B) {
+        if (HAS_PCH_IBX(dev_priv) && crtc->pipe == PIPE_B) {
                 /*
                  * We get CPU/PCH FIFO underruns on the other pipe when
                  * doing the workaround. Sweep them under the rug.
@@ -1241,7 +1241,7 @@ static enum drm_mode_status
 hdmi_port_clock_valid(struct intel_hdmi *hdmi,
                       int clock, bool respect_downstream_limits)
 {
-        struct drm_device *dev = intel_hdmi_to_dev(hdmi);
+        struct drm_i915_private *dev_priv = to_i915(intel_hdmi_to_dev(hdmi));
 
         if (clock < 25000)
                 return MODE_CLOCK_LOW;
@@ -1249,11 +1249,11 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi,
                 return MODE_CLOCK_HIGH;
 
         /* BXT DPLL can't generate 223-240 MHz */
-        if (IS_BROXTON(dev) && clock > 223333 && clock < 240000)
+        if (IS_BROXTON(dev_priv) && clock > 223333 && clock < 240000)
                 return MODE_CLOCK_RANGE;
 
         /* CHV DPLL can't generate 216-240 MHz */
-        if (IS_CHERRYVIEW(dev) && clock > 216000 && clock < 240000)
+        if (IS_CHERRYVIEW(dev_priv) && clock > 216000 && clock < 240000)
                 return MODE_CLOCK_RANGE;
 
         return MODE_OK;
@@ -1265,6 +1265,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 {
         struct intel_hdmi *hdmi = intel_attached_hdmi(connector);
         struct drm_device *dev = intel_hdmi_to_dev(hdmi);
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum drm_mode_status status;
         int clock;
         int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
@@ -1287,7 +1288,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
         status = hdmi_port_clock_valid(hdmi, clock, true);
 
         /* if we can't do 8bpc we may still be able to do 12bpc */
-        if (!HAS_GMCH_DISPLAY(dev) && status != MODE_OK)
+        if (!HAS_GMCH_DISPLAY(dev_priv) && status != MODE_OK)
                 status = hdmi_port_clock_valid(hdmi, clock * 3 / 2, true);
 
         return status;
@@ -1297,7 +1298,7 @@ static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc_state->base.crtc->dev;
 
-        if (HAS_GMCH_DISPLAY(dev))
+        if (HAS_GMCH_DISPLAY(to_i915(dev)))
                 return false;
 
         /*
@@ -1312,7 +1313,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
                                struct drm_connector_state *conn_state)
 {
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
         int clock_8bpc = pipe_config->base.adjusted_mode.crtc_clock;
         int clock_12bpc = clock_8bpc * 3 / 2;
@@ -1339,7 +1340,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
                 clock_12bpc *= 2;
         }
 
-        if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev))
+        if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv))
                 pipe_config->has_pch_encoder = true;
 
         if (pipe_config->has_hdmi_sink && intel_hdmi->has_audio)
@@ -1799,6 +1800,50 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *c
         intel_hdmi->aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
 }
 
+static u8 intel_hdmi_ddc_pin(struct drm_i915_private *dev_priv,
+                             enum port port)
+{
+        const struct ddi_vbt_port_info *info =
+                &dev_priv->vbt.ddi_port_info[port];
+        u8 ddc_pin;
+
+        if (info->alternate_ddc_pin) {
+                DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (VBT)\n",
+                              info->alternate_ddc_pin, port_name(port));
+                return info->alternate_ddc_pin;
+        }
+
+        switch (port) {
+        case PORT_B:
+                if (IS_BROXTON(dev_priv))
+                        ddc_pin = GMBUS_PIN_1_BXT;
+                else
+                        ddc_pin = GMBUS_PIN_DPB;
+                break;
+        case PORT_C:
+                if (IS_BROXTON(dev_priv))
+                        ddc_pin = GMBUS_PIN_2_BXT;
+                else
+                        ddc_pin = GMBUS_PIN_DPC;
+                break;
+        case PORT_D:
+                if (IS_CHERRYVIEW(dev_priv))
+                        ddc_pin = GMBUS_PIN_DPD_CHV;
+                else
+                        ddc_pin = GMBUS_PIN_DPD;
+                break;
+        default:
+                MISSING_CASE(port);
+                ddc_pin = GMBUS_PIN_DPB;
+                break;
+        }
+
+        DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (platform default)\n",
+                      ddc_pin, port_name(port));
+
+        return ddc_pin;
+}
+
 void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
                                struct intel_connector *intel_connector)
 {
@@ -1808,7 +1853,6 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
         struct drm_device *dev = intel_encoder->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
-        uint8_t alternate_ddc_pin;
 
         DRM_DEBUG_KMS("Adding HDMI connector on port %c\n",
                       port_name(port));
@@ -1826,12 +1870,10 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
         connector->doublescan_allowed = 0;
         connector->stereo_allowed = 1;
 
+        intel_hdmi->ddc_bus = intel_hdmi_ddc_pin(dev_priv, port);
+
         switch (port) {
         case PORT_B:
-                if (IS_BROXTON(dev_priv))
-                        intel_hdmi->ddc_bus = GMBUS_PIN_1_BXT;
-                else
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DPB;
                 /*
                  * On BXT A0/A1, sw needs to activate DDIA HPD logic and
                  * interrupts to check the external panel connection.
@@ -1842,61 +1884,32 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
                         intel_encoder->hpd_pin = HPD_PORT_B;
                 break;
         case PORT_C:
-                if (IS_BROXTON(dev_priv))
-                        intel_hdmi->ddc_bus = GMBUS_PIN_2_BXT;
-                else
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DPC;
                 intel_encoder->hpd_pin = HPD_PORT_C;
                 break;
         case PORT_D:
-                if (WARN_ON(IS_BROXTON(dev_priv)))
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DISABLED;
-                else if (IS_CHERRYVIEW(dev_priv))
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DPD_CHV;
-                else
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DPD;
                 intel_encoder->hpd_pin = HPD_PORT_D;
                 break;
         case PORT_E:
-                /* On SKL PORT E doesn't have seperate GMBUS pin
-                 *  We rely on VBT to set a proper alternate GMBUS pin. */
-                alternate_ddc_pin =
-                        dev_priv->vbt.ddi_port_info[PORT_E].alternate_ddc_pin;
-                switch (alternate_ddc_pin) {
-                case DDC_PIN_B:
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DPB;
-                        break;
-                case DDC_PIN_C:
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DPC;
-                        break;
-                case DDC_PIN_D:
-                        intel_hdmi->ddc_bus = GMBUS_PIN_DPD;
-                        break;
-                default:
-                        MISSING_CASE(alternate_ddc_pin);
-                }
                 intel_encoder->hpd_pin = HPD_PORT_E;
                 break;
-        case PORT_A:
-                intel_encoder->hpd_pin = HPD_PORT_A;
-                /* Internal port only for eDP. */
         default:
-                BUG();
+                MISSING_CASE(port);
+                return;
         }
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 intel_hdmi->write_infoframe = vlv_write_infoframe;
                 intel_hdmi->set_infoframes = vlv_set_infoframes;
                 intel_hdmi->infoframe_enabled = vlv_infoframe_enabled;
-        } else if (IS_G4X(dev)) {
+        } else if (IS_G4X(dev_priv)) {
                 intel_hdmi->write_infoframe = g4x_write_infoframe;
                 intel_hdmi->set_infoframes = g4x_set_infoframes;
                 intel_hdmi->infoframe_enabled = g4x_infoframe_enabled;
-        } else if (HAS_DDI(dev)) {
+        } else if (HAS_DDI(dev_priv)) {
                 intel_hdmi->write_infoframe = hsw_write_infoframe;
                 intel_hdmi->set_infoframes = hsw_set_infoframes;
                 intel_hdmi->infoframe_enabled = hsw_infoframe_enabled;
-        } else if (HAS_PCH_IBX(dev)) {
+        } else if (HAS_PCH_IBX(dev_priv)) {
                 intel_hdmi->write_infoframe = ibx_write_infoframe;
                 intel_hdmi->set_infoframes = ibx_set_infoframes;
                 intel_hdmi->infoframe_enabled = ibx_infoframe_enabled;
@@ -1906,7 +1919,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
                 intel_hdmi->infoframe_enabled = cpt_infoframe_enabled;
         }
 
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev_priv))
                 intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
         else
                 intel_connector->get_hw_state = intel_connector_get_hw_state;
@@ -1920,7 +1933,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
          * 0xd.  Failure to do so will result in spurious interrupts being
          * generated on the port when a cable is not attached.
          */
-        if (IS_G4X(dev) && !IS_GM45(dev)) {
+        if (IS_G4X(dev_priv) && !IS_GM45(dev_priv)) {
                 u32 temp = I915_READ(PEG_BAND_GAP_DATA);
                 I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
         }
@@ -1929,6 +1942,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 void intel_hdmi_init(struct drm_device *dev,
                      i915_reg_t hdmi_reg, enum port port)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_digital_port *intel_dig_port;
         struct intel_encoder *intel_encoder;
         struct intel_connector *intel_connector;
@@ -1949,7 +1963,7 @@ void intel_hdmi_init(struct drm_device *dev,
                          DRM_MODE_ENCODER_TMDS, "HDMI %c", port_name(port));
 
         intel_encoder->compute_config = intel_hdmi_compute_config;
-        if (HAS_PCH_SPLIT(dev)) {
+        if (HAS_PCH_SPLIT(dev_priv)) {
                 intel_encoder->disable = pch_disable_hdmi;
                 intel_encoder->post_disable = pch_post_disable_hdmi;
         } else {
@@ -1957,29 +1971,30 @@ void intel_hdmi_init(struct drm_device *dev,
         }
         intel_encoder->get_hw_state = intel_hdmi_get_hw_state;
         intel_encoder->get_config = intel_hdmi_get_config;
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(dev_priv)) {
                 intel_encoder->pre_pll_enable = chv_hdmi_pre_pll_enable;
                 intel_encoder->pre_enable = chv_hdmi_pre_enable;
                 intel_encoder->enable = vlv_enable_hdmi;
                 intel_encoder->post_disable = chv_hdmi_post_disable;
                 intel_encoder->post_pll_disable = chv_hdmi_post_pll_disable;
-        } else if (IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv)) {
                 intel_encoder->pre_pll_enable = vlv_hdmi_pre_pll_enable;
                 intel_encoder->pre_enable = vlv_hdmi_pre_enable;
                 intel_encoder->enable = vlv_enable_hdmi;
                 intel_encoder->post_disable = vlv_hdmi_post_disable;
         } else {
                 intel_encoder->pre_enable = intel_hdmi_pre_enable;
-                if (HAS_PCH_CPT(dev))
+                if (HAS_PCH_CPT(dev_priv))
                         intel_encoder->enable = cpt_enable_hdmi;
-                else if (HAS_PCH_IBX(dev))
+                else if (HAS_PCH_IBX(dev_priv))
                         intel_encoder->enable = ibx_enable_hdmi;
                 else
                         intel_encoder->enable = g4x_enable_hdmi;
         }
 
         intel_encoder->type = INTEL_OUTPUT_HDMI;
-        if (IS_CHERRYVIEW(dev)) {
+        intel_encoder->port = port;
+        if (IS_CHERRYVIEW(dev_priv)) {
                 if (port == PORT_D)
                         intel_encoder->crtc_mask = 1 << 2;
                 else
@@ -1993,7 +2008,7 @@ void intel_hdmi_init(struct drm_device *dev,
          * to work on real hardware. And since g4x can send infoframes to
          * only one port anyway, nothing is lost by allowing it.
          */
-        if (IS_G4X(dev))
+        if (IS_G4X(dev_priv))
                 intel_encoder->cloneable |= 1 << INTEL_OUTPUT_HDMI;
 
         intel_dig_port->port = port;
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 79aab9ad6faa..83f260bb4eef 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -138,11 +138,10 @@ static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable)
 static u32 get_reserved(struct intel_gmbus *bus)
 {
         struct drm_i915_private *dev_priv = bus->dev_priv;
-        struct drm_device *dev = &dev_priv->drm;
         u32 reserved = 0;
 
         /* On most chips, these bits must be preserved in software. */
-        if (!IS_I830(dev) && !IS_845G(dev))
+        if (!IS_I830(dev_priv) && !IS_845G(dev_priv))
                 reserved = I915_READ_NOTRACE(bus->gpio_reg) &
                                              (GPIO_DATA_PULLUP_DISABLE |
                                               GPIO_CLOCK_PULLUP_DISABLE);
@@ -468,13 +467,9 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
                                                struct intel_gmbus,
                                                adapter);
         struct drm_i915_private *dev_priv = bus->dev_priv;
-        const unsigned int fw =
-                intel_uncore_forcewake_for_reg(dev_priv, GMBUS0,
-                                               FW_REG_READ | FW_REG_WRITE);
         int i = 0, inc, try = 0;
         int ret = 0;
 
-        intel_uncore_forcewake_get(dev_priv, fw);
 retry:
         I915_WRITE_FW(GMBUS0, bus->reg0);
 
@@ -576,7 +571,6 @@ timeout:
         ret = -EAGAIN;
 
 out:
-        intel_uncore_forcewake_put(dev_priv, fw);
         return ret;
 }
 
@@ -633,10 +627,10 @@ int intel_setup_gmbus(struct drm_device *dev)
         unsigned int pin;
         int ret;
 
-        if (HAS_PCH_NOP(dev))
+        if (HAS_PCH_NOP(dev_priv))
                 return 0;
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 dev_priv->gpio_mmio_base = VLV_DISPLAY_BASE;
         else if (!HAS_GMCH_DISPLAY(dev_priv))
                 dev_priv->gpio_mmio_base =
@@ -674,7 +668,7 @@ int intel_setup_gmbus(struct drm_device *dev)
                 bus->reg0 = pin | GMBUS_RATE_100KHZ;
 
                 /* gmbus seems to be broken on i830 */
-                if (IS_I830(dev))
+                if (IS_I830(dev_priv))
                         bus->force_bit = 1;
 
                 intel_gpio_setup(bus, pin);
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 251143361f31..bc86585b9fbb 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -226,10 +226,16 @@ enum {
 /* Typical size of the average request (2 pipecontrols and a MI_BB) */
 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */
 
+#define WA_TAIL_DWORDS 2
+
 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
                                             struct intel_engine_cs *engine);
 static int intel_lr_context_pin(struct i915_gem_context *ctx,
                                 struct intel_engine_cs *engine);
+static void execlists_init_reg_state(u32 *reg_state,
+                                     struct i915_gem_context *ctx,
+                                     struct intel_engine_cs *engine,
+                                     struct intel_ring *ring);
 
 /**
  * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
@@ -269,8 +275,7 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
         struct drm_i915_private *dev_priv = engine->i915;
 
         engine->disable_lite_restore_wa =
-                (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
-                 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) &&
+                IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) &&
                 (engine->id == VCS || engine->id == VCS2);
 
         engine->ctx_desc_template = GEN8_CTX_VALID;
@@ -621,6 +626,10 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
 
         request->ring = ce->ring;
 
+        ret = intel_lr_context_pin(request->ctx, engine);
+        if (ret)
+                return ret;
+
         if (i915.enable_guc_submission) {
                 /*
                  * Check that the GuC has space for the request before
@@ -629,21 +638,17 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
                  */
                 ret = i915_guc_wq_reserve(request);
                 if (ret)
-                        return ret;
+                        goto err_unpin;
         }
 
-        ret = intel_lr_context_pin(request->ctx, engine);
-        if (ret)
-                return ret;
-
         ret = intel_ring_begin(request, 0);
         if (ret)
-                goto err_unpin;
+                goto err_unreserve;
 
         if (!ce->initialised) {
                 ret = engine->init_context(request);
                 if (ret)
-                        goto err_unpin;
+                        goto err_unreserve;
 
                 ce->initialised = true;
         }
@@ -658,6 +663,9 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
         request->reserved_space -= EXECLISTS_REQUEST_SIZE;
         return 0;
 
+err_unreserve:
+        if (i915.enable_guc_submission)
+                i915_guc_wq_unreserve(request);
 err_unpin:
         intel_lr_context_unpin(request->ctx, engine);
         return ret;
@@ -708,7 +716,6 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx,
 {
         struct intel_context *ce = &ctx->engine[engine->id];
         void *vaddr;
-        u32 *lrc_reg_state;
         int ret;
 
         lockdep_assert_held(&ctx->i915->drm.struct_mutex);
@@ -727,17 +734,16 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx,
                 goto unpin_vma;
         }
 
-        lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
-
         ret = intel_ring_pin(ce->ring);
         if (ret)
                 goto unpin_map;
 
         intel_lr_context_descriptor_update(ctx, engine);
 
-        lrc_reg_state[CTX_RING_BUFFER_START+1] =
+        ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
+        ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
                 i915_ggtt_offset(ce->ring->vma);
-        ce->lrc_reg_state = lrc_reg_state;
+
         ce->state->obj->dirty = true;
 
         /* Invalidate GuC TLB. */
@@ -846,13 +852,12 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
         uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES);
 
         /*
-         * WaDisableLSQCROPERFforOCL:skl,kbl
+         * WaDisableLSQCROPERFforOCL:kbl
          * This WA is implemented in skl_init_clock_gating() but since
          * this batch updates GEN8_L3SQCREG4 with default value we need to
          * set this bit here to retain the WA during flush.
          */
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0) ||
-            IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
+        if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
                 l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS;
 
         wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 |
@@ -995,9 +1000,8 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
         struct drm_i915_private *dev_priv = engine->i915;
         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
-        /* WaDisableCtxRestoreArbitration:skl,bxt */
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
-            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
+        /* WaDisableCtxRestoreArbitration:bxt */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
 
         /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */
@@ -1068,9 +1072,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
 {
         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
-        /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
-        if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_B0) ||
-            IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) {
+        /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
+        if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) {
                 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
                 wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
                 wa_ctx_emit(batch, index,
@@ -1097,9 +1100,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
                 wa_ctx_emit(batch, index, MI_NOOP);
         }
 
-        /* WaDisableCtxRestoreArbitration:skl,bxt */
-        if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_D0) ||
-            IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
+        /* WaDisableCtxRestoreArbitration:bxt */
+        if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
                 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
 
         wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
@@ -1231,7 +1233,7 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 
         lrc_init_hws(engine);
 
-        intel_engine_reset_irq(engine);
+        intel_engine_reset_breadcrumbs(engine);
 
         I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
 
@@ -1243,8 +1245,12 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 
         intel_engine_init_hangcheck(engine);
 
-        if (!execlists_elsp_idle(engine))
+        /* After a GPU reset, we may have requests to replay */
+        if (!execlists_elsp_idle(engine)) {
+                engine->execlist_port[0].count = 0;
+                engine->execlist_port[1].count = 0;
                 execlists_submit_ports(engine);
+        }
 
         return 0;
 }
@@ -1289,8 +1295,21 @@ static void reset_common_ring(struct intel_engine_cs *engine,
         struct execlist_port *port = engine->execlist_port;
         struct intel_context *ce = &request->ctx->engine[engine->id];
 
+        /* We want a simple context + ring to execute the breadcrumb update.
+         * We cannot rely on the context being intact across the GPU hang,
+         * so clear it and rebuild just what we need for the breadcrumb.
+         * All pending requests for this context will be zapped, and any
+         * future request will be after userspace has had the opportunity
+         * to recreate its own state.
+         */
+        execlists_init_reg_state(ce->lrc_reg_state,
+                                 request->ctx, engine, ce->ring);
+
         /* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
+        ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
+                i915_ggtt_offset(ce->ring->vma);
         ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix;
+
         request->ring->head = request->postfix;
         request->ring->last_retired_head = -1;
         intel_ring_update_space(request->ring);
@@ -1306,10 +1325,10 @@ static void reset_common_ring(struct intel_engine_cs *engine,
                 memset(&port[1], 0, sizeof(port[1]));
         }
 
-        /* CS is stopped, and we will resubmit both ports on resume */
         GEM_BUG_ON(request->ctx != port[0].request->ctx);
-        port[0].count = 0;
-        port[1].count = 0;
+
+        /* Reset WaIdleLiteRestore:bdw,skl as well */
+        request->tail = request->wa_tail - WA_TAIL_DWORDS * sizeof(u32);
 }
 
 static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
@@ -1547,7 +1566,6 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
  * used as a workaround for not being allowed to do lite
  * restore with HEAD==TAIL (WaIdleLiteRestore).
  */
-#define WA_TAIL_DWORDS 2
 
 static int gen8_emit_request(struct drm_i915_gem_request *request)
 {
@@ -1630,9 +1648,6 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 {
         struct drm_i915_private *dev_priv;
 
-        if (!intel_engine_initialized(engine))
-                return;
-
         /*
          * Tasklet cannot be active at this point due intel_mark_active/idle
          * so this is just for documentation.
@@ -1659,13 +1674,16 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 
         lrc_destroy_wa_ctx_obj(engine);
         engine->i915 = NULL;
+        dev_priv->engine[engine->id] = NULL;
+        kfree(engine);
 }
 
 void intel_execlists_enable_submission(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 engine->submit_request = execlists_submit_request;
 }
 
@@ -1894,38 +1912,13 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
         return indirect_ctx_offset;
 }
 
-static int
-populate_lr_context(struct i915_gem_context *ctx,
-                    struct drm_i915_gem_object *ctx_obj,
-                    struct intel_engine_cs *engine,
-                    struct intel_ring *ring)
+static void execlists_init_reg_state(u32 *reg_state,
+                                     struct i915_gem_context *ctx,
+                                     struct intel_engine_cs *engine,
+                                     struct intel_ring *ring)
 {
-        struct drm_i915_private *dev_priv = ctx->i915;
-        struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
-        void *vaddr;
-        u32 *reg_state;
-        int ret;
-
-        if (!ppgtt)
-                ppgtt = dev_priv->mm.aliasing_ppgtt;
-
-        ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
-        if (ret) {
-                DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
-                return ret;
-        }
-
-        vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
-        if (IS_ERR(vaddr)) {
-                ret = PTR_ERR(vaddr);
-                DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
-                return ret;
-        }
-        ctx_obj->dirty = true;
-
-        /* The second page of the context object contains some fields which must
-         * be set up prior to the first execution. */
-        reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
+        struct drm_i915_private *dev_priv = engine->i915;
+        struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt;
 
         /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
          * commands followed by (reg, value) pairs. The values we are setting here are
@@ -1939,19 +1932,16 @@ populate_lr_context(struct i915_gem_context *ctx,
                        _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
                                           CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
                                           (HAS_RESOURCE_STREAMER(dev_priv) ?
-                                            CTX_CTRL_RS_CTX_ENABLE : 0)));
+                                           CTX_CTRL_RS_CTX_ENABLE : 0)));
         ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base),
                        0);
         ASSIGN_CTX_REG(reg_state, CTX_RING_TAIL, RING_TAIL(engine->mmio_base),
                        0);
-        /* Ring buffer start address is not known until the buffer is pinned.
-         * It is written to the context image in execlists_update_context()
-         */
         ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_START,
                        RING_START(engine->mmio_base), 0);
         ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_CONTROL,
                        RING_CTL(engine->mmio_base),
-                       ((ring->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID);
+                       RING_CTL_SIZE(ring->size) | RING_VALID);
         ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_U,
                        RING_BBADDR_UDW(engine->mmio_base), 0);
         ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_L,
@@ -2029,6 +2019,36 @@ populate_lr_context(struct i915_gem_context *ctx,
                 ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
                                make_rpcs(dev_priv));
         }
+}
+
+static int
+populate_lr_context(struct i915_gem_context *ctx,
+                    struct drm_i915_gem_object *ctx_obj,
+                    struct intel_engine_cs *engine,
+                    struct intel_ring *ring)
+{
+        void *vaddr;
+        int ret;
+
+        ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
+        if (ret) {
+                DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
+                return ret;
+        }
+
+        vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
+        if (IS_ERR(vaddr)) {
+                ret = PTR_ERR(vaddr);
+                DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
+                return ret;
+        }
+        ctx_obj->dirty = true;
+
+        /* The second page of the context object contains some fields which must
+         * be set up prior to the first execution. */
+
+        execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE,
+                                 ctx, engine, ring);
 
         i915_gem_object_unpin_map(ctx_obj);
 
@@ -2129,30 +2149,43 @@ error_deref_obj:
 
 void intel_lr_context_resume(struct drm_i915_private *dev_priv)
 {
-        struct i915_gem_context *ctx = dev_priv->kernel_context;
         struct intel_engine_cs *engine;
+        struct i915_gem_context *ctx;
+        enum intel_engine_id id;
+
+        /* Because we emit WA_TAIL_DWORDS there may be a disparity
+         * between our bookkeeping in ce->ring->head and ce->ring->tail and
+         * that stored in context. As we only write new commands from
+         * ce->ring->tail onwards, everything before that is junk. If the GPU
+         * starts reading from its RING_HEAD from the context, it may try to
+         * execute that junk and die.
+         *
+         * So to avoid that we reset the context images upon resume. For
+         * simplicity, we just zero everything out.
+         */
+        list_for_each_entry(ctx, &dev_priv->context_list, link) {
+                for_each_engine(engine, dev_priv, id) {
+                        struct intel_context *ce = &ctx->engine[engine->id];
+                        u32 *reg;
 
-        for_each_engine(engine, dev_priv) {
-                struct intel_context *ce = &ctx->engine[engine->id];
-                void *vaddr;
-                uint32_t *reg_state;
-
-                if (!ce->state)
-                        continue;
-
-                vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
-                if (WARN_ON(IS_ERR(vaddr)))
-                        continue;
+                        if (!ce->state)
+                                continue;
 
-                reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
+                        reg = i915_gem_object_pin_map(ce->state->obj,
+                                                      I915_MAP_WB);
+                        if (WARN_ON(IS_ERR(reg)))
+                                continue;
 
-                reg_state[CTX_RING_HEAD+1] = 0;
-                reg_state[CTX_RING_TAIL+1] = 0;
+                        reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg);
+                        reg[CTX_RING_HEAD+1] = 0;
+                        reg[CTX_RING_TAIL+1] = 0;
 
-                ce->state->obj->dirty = true;
-                i915_gem_object_unpin_map(ce->state->obj);
+                        ce->state->obj->dirty = true;
+                        i915_gem_object_unpin_map(ce->state->obj);
 
-                ce->ring->head = 0;
-                ce->ring->tail = 0;
+                        ce->ring->head = ce->ring->tail = 0;
+                        ce->ring->last_retired_head = -1;
+                        intel_ring_update_space(ce->ring);
+                }
         }
 }
diff --git a/drivers/gpu/drm/i915/intel_lspcon.c b/drivers/gpu/drm/i915/intel_lspcon.c
new file mode 100644
index 000000000000..632149c6b3ad
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_lspcon.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+#include <drm/drm_edid.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_dp_dual_mode_helper.h>
+#include "intel_drv.h"
+
+static enum drm_lspcon_mode lspcon_get_current_mode(struct intel_lspcon *lspcon)
+{
+        enum drm_lspcon_mode current_mode = DRM_LSPCON_MODE_INVALID;
+        struct i2c_adapter *adapter = &lspcon->aux->ddc;
+
+        if (drm_lspcon_get_mode(adapter, &current_mode))
+                DRM_ERROR("Error reading LSPCON mode\n");
+        else
+                DRM_DEBUG_KMS("Current LSPCON mode %s\n",
+                        current_mode == DRM_LSPCON_MODE_PCON ? "PCON" : "LS");
+        return current_mode;
+}
+
+static int lspcon_change_mode(struct intel_lspcon *lspcon,
+        enum drm_lspcon_mode mode, bool force)
+{
+        int err;
+        enum drm_lspcon_mode current_mode;
+        struct i2c_adapter *adapter = &lspcon->aux->ddc;
+
+        err = drm_lspcon_get_mode(adapter, &current_mode);
+        if (err) {
+                DRM_ERROR("Error reading LSPCON mode\n");
+                return err;
+        }
+
+        if (current_mode == mode) {
+                DRM_DEBUG_KMS("Current mode = desired LSPCON mode\n");
+                return 0;
+        }
+
+        err = drm_lspcon_set_mode(adapter, mode);
+        if (err < 0) {
+                DRM_ERROR("LSPCON mode change failed\n");
+                return err;
+        }
+
+        lspcon->mode = mode;
+        DRM_DEBUG_KMS("LSPCON mode changed done\n");
+        return 0;
+}
+
+static bool lspcon_probe(struct intel_lspcon *lspcon)
+{
+        enum drm_dp_dual_mode_type adaptor_type;
+        struct i2c_adapter *adapter = &lspcon->aux->ddc;
+
+        /* Lets probe the adaptor and check its type */
+        adaptor_type = drm_dp_dual_mode_detect(adapter);
+        if (adaptor_type != DRM_DP_DUAL_MODE_LSPCON) {
+                DRM_DEBUG_KMS("No LSPCON detected, found %s\n",
+                        drm_dp_get_dual_mode_type_name(adaptor_type));
+                return false;
+        }
+
+        /* Yay ... got a LSPCON device */
+        DRM_DEBUG_KMS("LSPCON detected\n");
+        lspcon->mode = lspcon_get_current_mode(lspcon);
+        lspcon->active = true;
+        return true;
+}
+
+void lspcon_resume(struct intel_lspcon *lspcon)
+{
+        if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON, true))
+                DRM_ERROR("LSPCON resume failed\n");
+        else
+                DRM_DEBUG_KMS("LSPCON resume success\n");
+}
+
+bool lspcon_init(struct intel_digital_port *intel_dig_port)
+{
+        struct intel_dp *dp = &intel_dig_port->dp;
+        struct intel_lspcon *lspcon = &intel_dig_port->lspcon;
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        if (!IS_GEN9(dev_priv)) {
+                DRM_ERROR("LSPCON is supported on GEN9 only\n");
+                return false;
+        }
+
+        lspcon->active = false;
+        lspcon->mode = DRM_LSPCON_MODE_INVALID;
+        lspcon->aux = &dp->aux;
+
+        if (!lspcon_probe(lspcon)) {
+                DRM_ERROR("Failed to probe lspcon\n");
+                return false;
+        }
+
+        /*
+        * In the SW state machine, lets Put LSPCON in PCON mode only.
+        * In this way, it will work with both HDMI 1.4 sinks as well as HDMI
+        * 2.0 sinks.
+        */
+        if (lspcon->active && lspcon->mode != DRM_LSPCON_MODE_PCON) {
+                if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON,
+                        true) < 0) {
+                        DRM_ERROR("LSPCON mode change to PCON failed\n");
+                        return false;
+                }
+        }
+
+        DRM_DEBUG_KMS("Success: LSPCON init\n");
+        return true;
+}
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index e1d47d51ea47..199b90c7907a 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -106,7 +106,7 @@ static bool intel_lvds_get_hw_state(struct intel_encoder *encoder,
         if (!(tmp & LVDS_PORT_EN))
                 goto out;
 
-        if (HAS_PCH_CPT(dev))
+        if (HAS_PCH_CPT(dev_priv))
                 *pipe = PORT_TO_PIPE_CPT(tmp);
         else
                 *pipe = PORT_TO_PIPE(tmp);
@@ -396,7 +396,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
                                       struct intel_crtc_state *pipe_config,
                                       struct drm_connector_state *conn_state)
 {
-        struct drm_device *dev = intel_encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
         struct intel_lvds_encoder *lvds_encoder =
                 to_lvds_encoder(&intel_encoder->base);
         struct intel_connector *intel_connector =
@@ -406,7 +406,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
         unsigned int lvds_bpp;
 
         /* Should never happen!! */
-        if (INTEL_INFO(dev)->gen < 4 && intel_crtc->pipe == 0) {
+        if (INTEL_GEN(dev_priv) < 4 && intel_crtc->pipe == 0) {
                 DRM_ERROR("Can't support LVDS on pipe A\n");
                 return false;
         }
@@ -431,7 +431,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
         intel_fixed_panel_mode(intel_connector->panel.fixed_mode,
                                adjusted_mode);
 
-        if (HAS_PCH_SPLIT(dev)) {
+        if (HAS_PCH_SPLIT(dev_priv)) {
                 pipe_config->has_pch_encoder = true;
 
                 intel_pch_panel_fitting(intel_crtc, pipe_config,
@@ -566,7 +566,7 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
          * and as part of the cleanup in the hw state restore we also redisable
          * the vga plane.
          */
-        if (!HAS_PCH_SPLIT(dev))
+        if (!HAS_PCH_SPLIT(dev_priv))
                 intel_display_resume(dev);
 
         dev_priv->modeset_restore = MODESET_DONE;
@@ -949,16 +949,17 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
         return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP;
 }
 
-static bool intel_lvds_supported(struct drm_device *dev)
+static bool intel_lvds_supported(struct drm_i915_private *dev_priv)
 {
         /* With the introduction of the PCH we gained a dedicated
          * LVDS presence pin, use it. */
-        if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+        if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
                 return true;
 
         /* Otherwise LVDS was only attached to mobile products,
          * except for the inglorious 830gm */
-        if (INTEL_INFO(dev)->gen <= 4 && IS_MOBILE(dev) && !IS_I830(dev))
+        if (INTEL_GEN(dev_priv) <= 4 &&
+            IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
                 return true;
 
         return false;
@@ -990,21 +991,21 @@ void intel_lvds_init(struct drm_device *dev)
         int pipe;
         u8 pin;
 
-        if (!intel_lvds_supported(dev))
+        if (!intel_lvds_supported(dev_priv))
                 return;
 
         /* Skip init on machines we know falsely report LVDS */
         if (dmi_check_system(intel_no_lvds))
                 return;
 
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 lvds_reg = PCH_LVDS;
         else
                 lvds_reg = LVDS;
 
         lvds = I915_READ(lvds_reg);
 
-        if (HAS_PCH_SPLIT(dev)) {
+        if (HAS_PCH_SPLIT(dev_priv)) {
                 if ((lvds & LVDS_DETECTED) == 0)
                         return;
                 if (dev_priv->vbt.edp.support) {
@@ -1064,12 +1065,13 @@ void intel_lvds_init(struct drm_device *dev)
         intel_connector->get_hw_state = intel_connector_get_hw_state;
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
-        intel_encoder->type = INTEL_OUTPUT_LVDS;
 
+        intel_encoder->type = INTEL_OUTPUT_LVDS;
+        intel_encoder->port = PORT_NONE;
         intel_encoder->cloneable = 0;
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
-        else if (IS_GEN4(dev))
+        else if (IS_GEN4(dev_priv))
                 intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
         else
                 intel_encoder->crtc_mask = (1 << 1);
@@ -1157,7 +1159,7 @@ void intel_lvds_init(struct drm_device *dev)
          */
 
         /* Ironlake: FIXME if still fail, not try pipe mode now */
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev_priv))
                 goto failed;
 
         pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0;
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index a24bc8c7889f..25bcd4a178d3 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -233,7 +233,7 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
 static struct drm_i915_gem_request *alloc_request(struct intel_overlay *overlay)
 {
         struct drm_i915_private *dev_priv = overlay->i915;
-        struct intel_engine_cs *engine = &dev_priv->engine[RCS];
+        struct intel_engine_cs *engine = dev_priv->engine[RCS];
 
         return i915_gem_request_alloc(engine, dev_priv->kernel_context);
 }
@@ -1470,6 +1470,8 @@ void intel_cleanup_overlay(struct drm_i915_private *dev_priv)
         kfree(dev_priv->overlay);
 }
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 struct intel_overlay_error_state {
         struct overlay_registers regs;
         unsigned long base;
@@ -1587,3 +1589,5 @@ intel_overlay_print_error_state(struct drm_i915_error_state_buf *m,
         P(UVSCALEV);
 #undef P
 }
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index c10e9b0405e8..be4b4d546fd9 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -841,7 +841,7 @@ static void lpt_enable_backlight(struct intel_connector *connector)
 {
         struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
         struct intel_panel *panel = &connector->panel;
-        u32 pch_ctl1, pch_ctl2;
+        u32 pch_ctl1, pch_ctl2, schicken;
 
         pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
         if (pch_ctl1 & BLM_PCH_PWM_ENABLE) {
@@ -850,6 +850,22 @@ static void lpt_enable_backlight(struct intel_connector *connector)
                 I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1);
         }
 
+        if (HAS_PCH_LPT(dev_priv)) {
+                schicken = I915_READ(SOUTH_CHICKEN2);
+                if (panel->backlight.alternate_pwm_increment)
+                        schicken |= LPT_PWM_GRANULARITY;
+                else
+                        schicken &= ~LPT_PWM_GRANULARITY;
+                I915_WRITE(SOUTH_CHICKEN2, schicken);
+        } else {
+                schicken = I915_READ(SOUTH_CHICKEN1);
+                if (panel->backlight.alternate_pwm_increment)
+                        schicken |= SPT_PWM_GRANULARITY;
+                else
+                        schicken &= ~SPT_PWM_GRANULARITY;
+                I915_WRITE(SOUTH_CHICKEN1, schicken);
+        }
+
         pch_ctl2 = panel->backlight.max << 16;
         I915_WRITE(BLC_PWM_PCH_CTL2, pch_ctl2);
 
@@ -1242,10 +1258,10 @@ static u32 bxt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
  */
 static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
-        struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+        struct intel_panel *panel = &connector->panel;
         u32 mul;
 
-        if (I915_READ(SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY)
+        if (panel->backlight.alternate_pwm_increment)
                 mul = 128;
         else
                 mul = 16;
@@ -1261,9 +1277,10 @@ static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
         struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+        struct intel_panel *panel = &connector->panel;
         u32 mul, clock;
 
-        if (I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY)
+        if (panel->backlight.alternate_pwm_increment)
                 mul = 16;
         else
                 mul = 128;
@@ -1414,6 +1431,13 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus
         struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
         struct intel_panel *panel = &connector->panel;
         u32 pch_ctl1, pch_ctl2, val;
+        bool alt;
+
+        if (HAS_PCH_LPT(dev_priv))
+                alt = I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY;
+        else
+                alt = I915_READ(SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY;
+        panel->backlight.alternate_pwm_increment = alt;
 
         pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
         panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 1472400ddce3..560fc7af8267 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -252,8 +252,8 @@ static const struct cxsr_latency cxsr_latency_table[] = {
         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 };
 
-static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
-                                                         int is_ddr3,
+static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
+                                                         bool is_ddr3,
                                                          int fsb,
                                                          int mem)
 {
@@ -322,11 +322,11 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
         struct drm_device *dev = &dev_priv->drm;
         u32 val;
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
                 POSTING_READ(FW_BLC_SELF_VLV);
                 dev_priv->wm.vlv.cxsr = enable;
-        } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
+        } else if (IS_G4X(dev_priv) || IS_CRESTLINE(dev_priv)) {
                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
                 POSTING_READ(FW_BLC_SELF);
         } else if (IS_PINEVIEW(dev)) {
@@ -334,12 +334,12 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
                 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
                 I915_WRITE(DSPFW3, val);
                 POSTING_READ(DSPFW3);
-        } else if (IS_I945G(dev) || IS_I945GM(dev)) {
+        } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
                 I915_WRITE(FW_BLC_SELF, val);
                 POSTING_READ(FW_BLC_SELF);
-        } else if (IS_I915GM(dev)) {
+        } else if (IS_I915GM(dev_priv)) {
                 /*
                  * FIXME can't find a bit like this for 915G, and
                  * and yet it does have the related watermark in
@@ -648,8 +648,10 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc)
         u32 reg;
         unsigned long wm;
 
-        latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
-                                         dev_priv->fsb_freq, dev_priv->mem_freq);
+        latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
+                                         dev_priv->is_ddr3,
+                                         dev_priv->fsb_freq,
+                                         dev_priv->mem_freq);
         if (!latency) {
                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
                 intel_set_memory_cxsr(dev_priv, false);
@@ -775,13 +777,13 @@ static bool g4x_check_srwm(struct drm_device *dev,
                       display_wm, cursor_wm);
 
         if (display_wm > display->max_wm) {
-                DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
+                DRM_DEBUG_KMS("display watermark is too large(%d/%u), disabling\n",
                               display_wm, display->max_wm);
                 return false;
         }
 
         if (cursor_wm > cursor->max_wm) {
-                DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
+                DRM_DEBUG_KMS("cursor watermark is too large(%d/%u), disabling\n",
                               cursor_wm, cursor->max_wm);
                 return false;
         }
@@ -1528,7 +1530,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 
         if (IS_I945GM(dev))
                 wm_info = &i945_wm_info;
-        else if (!IS_GEN2(dev))
+        else if (!IS_GEN2(dev_priv))
                 wm_info = &i915_wm_info;
         else
                 wm_info = &i830_a_wm_info;
@@ -1538,7 +1540,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
         if (intel_crtc_active(crtc)) {
                 const struct drm_display_mode *adjusted_mode;
                 int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
-                if (IS_GEN2(dev))
+                if (IS_GEN2(dev_priv))
                         cpp = 4;
 
                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
@@ -1552,7 +1554,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
                         planea_wm = wm_info->max_wm;
         }
 
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 wm_info = &i830_bc_wm_info;
 
         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
@@ -1560,7 +1562,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
         if (intel_crtc_active(crtc)) {
                 const struct drm_display_mode *adjusted_mode;
                 int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
-                if (IS_GEN2(dev))
+                if (IS_GEN2(dev_priv))
                         cpp = 4;
 
                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
@@ -1579,7 +1581,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 
         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
 
-        if (IS_I915GM(dev) && enabled) {
+        if (IS_I915GM(dev_priv) && enabled) {
                 struct drm_i915_gem_object *obj;
 
                 obj = intel_fb_obj(enabled->primary->state->fb);
@@ -1609,7 +1611,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
                 unsigned long line_time_us;
                 int entries;
 
-                if (IS_I915GM(dev) || IS_I945GM(dev))
+                if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
                         cpp = 4;
 
                 line_time_us = max(htotal * 1000 / clock, 1);
@@ -1623,7 +1625,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
                 if (srwm < 0)
                         srwm = 1;
 
-                if (IS_I945G(dev) || IS_I945GM(dev))
+                if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
                         I915_WRITE(FW_BLC_SELF,
                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
                 else
@@ -2080,10 +2082,10 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (IS_GEN9(dev)) {
+        if (IS_GEN9(dev_priv)) {
                 uint32_t val;
                 int ret, i;
-                int level, max_level = ilk_wm_max_level(dev);
+                int level, max_level = ilk_wm_max_level(dev_priv);
 
                 /* read the first set of memory latencies[0:3] */
                 val = 0; /* data0 to be programmed to 0 for first set */
@@ -2127,33 +2129,35 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
                                 GEN9_MEM_LATENCY_LEVEL_MASK;
 
                 /*
+                 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
+                 * need to be disabled. We make sure to sanitize the values out
+                 * of the punit to satisfy this requirement.
+                 */
+                for (level = 1; level <= max_level; level++) {
+                        if (wm[level] == 0) {
+                                for (i = level + 1; i <= max_level; i++)
+                                        wm[i] = 0;
+                                break;
+                        }
+                }
+
+                /*
                  * WaWmMemoryReadLatency:skl
                  *
                  * punit doesn't take into account the read latency so we need
-                 * to add 2us to the various latency levels we retrieve from
-                 * the punit.
-                 *   - W0 is a bit special in that it's the only level that
-                 *   can't be disabled if we want to have display working, so
-                 *   we always add 2us there.
-                 *   - For levels >=1, punit returns 0us latency when they are
-                 *   disabled, so we respect that and don't add 2us then
-                 *
-                 * Additionally, if a level n (n > 1) has a 0us latency, all
-                 * levels m (m >= n) need to be disabled. We make sure to
-                 * sanitize the values out of the punit to satisfy this
-                 * requirement.
+                 * to add 2us to the various latency levels we retrieve from the
+                 * punit when level 0 response data us 0us.
                  */
-                wm[0] += 2;
-                for (level = 1; level <= max_level; level++)
-                        if (wm[level] != 0)
+                if (wm[0] == 0) {
+                        wm[0] += 2;
+                        for (level = 1; level <= max_level; level++) {
+                                if (wm[level] == 0)
+                                        break;
                                 wm[level] += 2;
-                        else {
-                                for (i = level + 1; i <= max_level; i++)
-                                        wm[i] = 0;
-
-                                break;
                         }
-        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+                }
+
+        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
 
                 wm[0] = (sskpd >> 56) & 0xFF;
@@ -2180,42 +2184,44 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
         }
 }
 
-static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
+static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
+                                       uint16_t wm[5])
 {
         /* ILK sprite LP0 latency is 1300 ns */
-        if (IS_GEN5(dev))
+        if (IS_GEN5(dev_priv))
                 wm[0] = 13;
 }
 
-static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
+static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
+                                       uint16_t wm[5])
 {
         /* ILK cursor LP0 latency is 1300 ns */
-        if (IS_GEN5(dev))
+        if (IS_GEN5(dev_priv))
                 wm[0] = 13;
 
         /* WaDoubleCursorLP3Latency:ivb */
-        if (IS_IVYBRIDGE(dev))
+        if (IS_IVYBRIDGE(dev_priv))
                 wm[3] *= 2;
 }
 
-int ilk_wm_max_level(const struct drm_device *dev)
+int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
 {
         /* how many WM levels are we expecting */
-        if (INTEL_INFO(dev)->gen >= 9)
+        if (INTEL_GEN(dev_priv) >= 9)
                 return 7;
-        else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 return 4;
-        else if (INTEL_INFO(dev)->gen >= 6)
+        else if (INTEL_GEN(dev_priv) >= 6)
                 return 3;
         else
                 return 2;
 }
 
-static void intel_print_wm_latency(struct drm_device *dev,
+static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
                                    const char *name,
                                    const uint16_t wm[8])
 {
-        int level, max_level = ilk_wm_max_level(dev);
+        int level, max_level = ilk_wm_max_level(dev_priv);
 
         for (level = 0; level <= max_level; level++) {
                 unsigned int latency = wm[level];
@@ -2230,7 +2236,7 @@ static void intel_print_wm_latency(struct drm_device *dev,
                  * - latencies are in us on gen9.
                  * - before then, WM1+ latency values are in 0.5us units
                  */
-                if (IS_GEN9(dev))
+                if (IS_GEN9(dev_priv))
                         latency *= 10;
                 else if (level > 0)
                         latency *= 5;
@@ -2244,7 +2250,7 @@ static void intel_print_wm_latency(struct drm_device *dev,
 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
                                     uint16_t wm[5], uint16_t min)
 {
-        int level, max_level = ilk_wm_max_level(&dev_priv->drm);
+        int level, max_level = ilk_wm_max_level(dev_priv);
 
         if (wm[0] >= min)
                 return false;
@@ -2273,9 +2279,9 @@ static void snb_wm_latency_quirk(struct drm_device *dev)
                 return;
 
         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
-        intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
-        intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
-        intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
+        intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
+        intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
+        intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
 }
 
 static void ilk_setup_wm_latency(struct drm_device *dev)
@@ -2289,14 +2295,14 @@ static void ilk_setup_wm_latency(struct drm_device *dev)
         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
                sizeof(dev_priv->wm.pri_latency));
 
-        intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
-        intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
+        intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
+        intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
 
-        intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
-        intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
-        intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
+        intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
+        intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
+        intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
 
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev_priv))
                 snb_wm_latency_quirk(dev);
 }
 
@@ -2305,7 +2311,7 @@ static void skl_setup_wm_latency(struct drm_device *dev)
         struct drm_i915_private *dev_priv = to_i915(dev);
 
         intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
-        intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
+        intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
 }
 
 static bool ilk_validate_pipe_wm(struct drm_device *dev,
@@ -2343,7 +2349,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
         struct intel_plane_state *pristate = NULL;
         struct intel_plane_state *sprstate = NULL;
         struct intel_plane_state *curstate = NULL;
-        int level, max_level = ilk_wm_max_level(dev), usable_level;
+        int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
         struct ilk_wm_maximums max;
 
         pipe_wm = &cstate->wm.ilk.optimal;
@@ -2388,7 +2394,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
         pipe_wm->wm[0] = pipe_wm->raw_wm[0];
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
 
         if (!ilk_validate_pipe_wm(dev, pipe_wm))
@@ -2430,7 +2436,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
 {
         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
         struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk;
-        int level, max_level = ilk_wm_max_level(dev);
+        int level, max_level = ilk_wm_max_level(to_i915(dev));
 
         /*
          * Start with the final, target watermarks, then combine with the
@@ -2514,11 +2520,11 @@ static void ilk_wm_merge(struct drm_device *dev,
                          struct intel_pipe_wm *merged)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
-        int level, max_level = ilk_wm_max_level(dev);
+        int level, max_level = ilk_wm_max_level(dev_priv);
         int last_enabled_level = max_level;
 
         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
-        if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
+        if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
             config->num_pipes_active > 1)
                 last_enabled_level = 0;
 
@@ -2554,7 +2560,7 @@ static void ilk_wm_merge(struct drm_device *dev,
          * What we should check here is whether FBC can be
          * enabled sometime later.
          */
-        if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
+        if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
             intel_fbc_is_active(dev_priv)) {
                 for (level = 2; level <= max_level; level++) {
                         struct intel_wm_level *wm = &merged->wm[level];
@@ -2575,7 +2581,7 @@ static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 return 2 * level;
         else
                 return dev_priv->wm.pri_latency[level];
@@ -2654,7 +2660,7 @@ static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
                                                   struct intel_pipe_wm *r1,
                                                   struct intel_pipe_wm *r2)
 {
-        int level, max_level = ilk_wm_max_level(dev);
+        int level, max_level = ilk_wm_max_level(to_i915(dev));
         int level1 = 0, level2 = 0;
 
         for (level = 1; level <= max_level; level++) {
@@ -2799,7 +2805,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
 
         if (dirty & WM_DIRTY_DDB) {
-                if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+                if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                         val = I915_READ(WM_MISC);
                         if (results->partitioning == INTEL_DDB_PART_1_2)
                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
@@ -2878,6 +2884,34 @@ skl_wm_plane_id(const struct intel_plane *plane)
 }
 
 /*
+ * FIXME: We still don't have the proper code detect if we need to apply the WA,
+ * so assume we'll always need it in order to avoid underruns.
+ */
+static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
+{
+        struct drm_i915_private *dev_priv = to_i915(state->base.dev);
+
+        if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
+            IS_KABYLAKE(dev_priv))
+                return true;
+
+        return false;
+}
+
+static bool
+intel_has_sagv(struct drm_i915_private *dev_priv)
+{
+        if (IS_KABYLAKE(dev_priv))
+                return true;
+
+        if (IS_SKYLAKE(dev_priv) &&
+            dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
+                return true;
+
+        return false;
+}
+
+/*
  * SAGV dynamically adjusts the system agent voltage and clock frequencies
  * depending on power and performance requirements. The display engine access
  * to system memory is blocked during the adjustment time. Because of the
@@ -2889,12 +2923,14 @@ skl_wm_plane_id(const struct intel_plane *plane)
  *  - We're not using an interlaced display configuration
  */
 int
-skl_enable_sagv(struct drm_i915_private *dev_priv)
+intel_enable_sagv(struct drm_i915_private *dev_priv)
 {
         int ret;
 
-        if (dev_priv->skl_sagv_status == I915_SKL_SAGV_NOT_CONTROLLED ||
-            dev_priv->skl_sagv_status == I915_SKL_SAGV_ENABLED)
+        if (!intel_has_sagv(dev_priv))
+                return 0;
+
+        if (dev_priv->sagv_status == I915_SAGV_ENABLED)
                 return 0;
 
         DRM_DEBUG_KMS("Enabling the SAGV\n");
@@ -2910,21 +2946,21 @@ skl_enable_sagv(struct drm_i915_private *dev_priv)
          * Some skl systems, pre-release machines in particular,
          * don't actually have an SAGV.
          */
-        if (ret == -ENXIO) {
+        if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
-                dev_priv->skl_sagv_status = I915_SKL_SAGV_NOT_CONTROLLED;
+                dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
                 return 0;
         } else if (ret < 0) {
                 DRM_ERROR("Failed to enable the SAGV\n");
                 return ret;
         }
 
-        dev_priv->skl_sagv_status = I915_SKL_SAGV_ENABLED;
+        dev_priv->sagv_status = I915_SAGV_ENABLED;
         return 0;
 }
 
 static int
-skl_do_sagv_disable(struct drm_i915_private *dev_priv)
+intel_do_sagv_disable(struct drm_i915_private *dev_priv)
 {
         int ret;
         uint32_t temp = GEN9_SAGV_DISABLE;
@@ -2938,19 +2974,21 @@ skl_do_sagv_disable(struct drm_i915_private *dev_priv)
 }
 
 int
-skl_disable_sagv(struct drm_i915_private *dev_priv)
+intel_disable_sagv(struct drm_i915_private *dev_priv)
 {
         int ret, result;
 
-        if (dev_priv->skl_sagv_status == I915_SKL_SAGV_NOT_CONTROLLED ||
-            dev_priv->skl_sagv_status == I915_SKL_SAGV_DISABLED)
+        if (!intel_has_sagv(dev_priv))
+                return 0;
+
+        if (dev_priv->sagv_status == I915_SAGV_DISABLED)
                 return 0;
 
         DRM_DEBUG_KMS("Disabling the SAGV\n");
         mutex_lock(&dev_priv->rps.hw_lock);
 
         /* bspec says to keep retrying for at least 1 ms */
-        ret = wait_for(result = skl_do_sagv_disable(dev_priv), 1);
+        ret = wait_for(result = intel_do_sagv_disable(dev_priv), 1);
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         if (ret == -ETIMEDOUT) {
@@ -2962,27 +3000,33 @@ skl_disable_sagv(struct drm_i915_private *dev_priv)
          * Some skl systems, pre-release machines in particular,
          * don't actually have an SAGV.
          */
-        if (result == -ENXIO) {
+        if (IS_SKYLAKE(dev_priv) && result == -ENXIO) {
                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
-                dev_priv->skl_sagv_status = I915_SKL_SAGV_NOT_CONTROLLED;
+                dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
                 return 0;
         } else if (result < 0) {
                 DRM_ERROR("Failed to disable the SAGV\n");
                 return result;
         }
 
-        dev_priv->skl_sagv_status = I915_SKL_SAGV_DISABLED;
+        dev_priv->sagv_status = I915_SAGV_DISABLED;
         return 0;
 }
 
-bool skl_can_enable_sagv(struct drm_atomic_state *state)
+bool intel_can_enable_sagv(struct drm_atomic_state *state)
 {
         struct drm_device *dev = state->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-        struct drm_crtc *crtc;
+        struct intel_crtc *crtc;
+        struct intel_plane *plane;
+        struct intel_crtc_state *cstate;
+        struct skl_plane_wm *wm;
         enum pipe pipe;
-        int level, plane;
+        int level, latency;
+
+        if (!intel_has_sagv(dev_priv))
+                return false;
 
         /*
          * SKL workaround: bspec recommends we disable the SAGV when we have
@@ -2997,27 +3041,37 @@ bool skl_can_enable_sagv(struct drm_atomic_state *state)
 
         /* Since we're now guaranteed to only have one active CRTC... */
         pipe = ffs(intel_state->active_crtcs) - 1;
-        crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+        crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+        cstate = to_intel_crtc_state(crtc->base.state);
 
-        if (crtc->state->mode.flags & DRM_MODE_FLAG_INTERLACE)
+        if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
                 return false;
 
-        for_each_plane(dev_priv, pipe, plane) {
+        for_each_intel_plane_on_crtc(dev, crtc, plane) {
+                wm = &cstate->wm.skl.optimal.planes[skl_wm_plane_id(plane)];
+
                 /* Skip this plane if it's not enabled */
-                if (intel_state->wm_results.plane[pipe][plane][0] == 0)
+                if (!wm->wm[0].plane_en)
                         continue;
 
                 /* Find the highest enabled wm level for this plane */
-                for (level = ilk_wm_max_level(dev);
-                     intel_state->wm_results.plane[pipe][plane][level] == 0; --level)
+                for (level = ilk_wm_max_level(dev_priv);
+                     !wm->wm[level].plane_en; --level)
                      { }
 
+                latency = dev_priv->wm.skl_latency[level];
+
+                if (skl_needs_memory_bw_wa(intel_state) &&
+                    plane->base.state->fb->modifier[0] ==
+                    I915_FORMAT_MOD_X_TILED)
+                        latency += 15;
+
                 /*
                  * If any of the planes on this pipe don't enable wm levels
                  * that incur memory latencies higher then 30µs we can't enable
                  * the SAGV
                  */
-                if (dev_priv->wm.skl_latency[level] < SKL_SAGV_BLOCK_TIME)
+                if (latency < SKL_SAGV_BLOCK_TIME)
                         return false;
         }
 
@@ -3036,7 +3090,6 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
         struct drm_crtc *for_crtc = cstate->base.crtc;
         unsigned int pipe_size, ddb_size;
         int nth_active_pipe;
-        int pipe = to_intel_crtc(for_crtc)->pipe;
 
         if (WARN_ON(!state) || !cstate->base.active) {
                 alloc->start = 0;
@@ -3064,7 +3117,7 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
          * we currently hold.
          */
         if (!intel_state->active_pipe_changes) {
-                *alloc = dev_priv->wm.skl_hw.ddb.pipe[pipe];
+                *alloc = to_intel_crtc(for_crtc)->hw_ddb;
                 return;
         }
 
@@ -3332,7 +3385,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
         struct drm_plane *plane;
         struct drm_plane_state *pstate;
         enum pipe pipe = intel_crtc->pipe;
-        struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
+        struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
         uint16_t alloc_size, start, cursor_blocks;
         uint16_t *minimum = cstate->wm.skl.minimum_blocks;
         uint16_t *y_minimum = cstate->wm.skl.minimum_y_blocks;
@@ -3340,13 +3393,15 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
         int num_active;
         int id, i;
 
+        /* Clear the partitioning for disabled planes. */
+        memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
+        memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe]));
+
         if (WARN_ON(!state))
                 return 0;
 
         if (!cstate->base.active) {
-                ddb->pipe[pipe].start = ddb->pipe[pipe].end = 0;
-                memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
-                memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe]));
+                alloc->start = alloc->end = 0;
                 return 0;
         }
 
@@ -3446,12 +3501,6 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
         return 0;
 }
 
-static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
-{
-        /* TODO: Take into account the scalers once we support them */
-        return config->base.adjusted_mode.crtc_clock;
-}
-
 /*
  * The max latency should be 257 (max the punit can code is 255 and we add 2us
  * for the read latency) and cpp should always be <= 8, so that
@@ -3472,29 +3521,14 @@ static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latenc
 }
 
 static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
-                               uint32_t horiz_pixels, uint8_t cpp,
-                               uint64_t tiling, uint32_t latency)
+                               uint32_t latency, uint32_t plane_blocks_per_line)
 {
         uint32_t ret;
-        uint32_t plane_bytes_per_line, plane_blocks_per_line;
         uint32_t wm_intermediate_val;
 
         if (latency == 0)
                 return UINT_MAX;
 
-        plane_bytes_per_line = horiz_pixels * cpp;
-
-        if (tiling == I915_FORMAT_MOD_Y_TILED ||
-            tiling == I915_FORMAT_MOD_Yf_TILED) {
-                plane_bytes_per_line *= 4;
-                plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
-                plane_blocks_per_line /= 4;
-        } else if (tiling == DRM_FORMAT_MOD_NONE) {
-                plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512) + 1;
-        } else {
-                plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
-        }
-
         wm_intermediate_val = latency * pixel_rate;
         ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
                                 plane_blocks_per_line;
@@ -3517,7 +3551,7 @@ static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cst
          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
          * with additional adjustments for plane-specific scaling.
          */
-        adjusted_pixel_rate = skl_pipe_pixel_rate(cstate);
+        adjusted_pixel_rate = ilk_pipe_pixel_rate(cstate);
         downscale_amount = skl_plane_downscale_amount(pstate);
 
         pixel_rate = adjusted_pixel_rate * downscale_amount >> 16;
@@ -3545,12 +3579,19 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
         uint8_t cpp;
         uint32_t width = 0, height = 0;
         uint32_t plane_pixel_rate;
+        uint32_t y_tile_minimum, y_min_scanlines;
+        struct intel_atomic_state *state =
+                to_intel_atomic_state(cstate->base.state);
+        bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
 
         if (latency == 0 || !cstate->base.active || !intel_pstate->base.visible) {
                 *enabled = false;
                 return 0;
         }
 
+        if (apply_memory_bw_wa && fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+                latency += 15;
+
         width = drm_rect_width(&intel_pstate->base.src) >> 16;
         height = drm_rect_height(&intel_pstate->base.src) >> 16;
 
@@ -3560,41 +3601,60 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
         cpp = drm_format_plane_cpp(fb->pixel_format, 0);
         plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
 
+        if (drm_rotation_90_or_270(pstate->rotation)) {
+                int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
+                        drm_format_plane_cpp(fb->pixel_format, 1) :
+                        drm_format_plane_cpp(fb->pixel_format, 0);
+
+                switch (cpp) {
+                case 1:
+                        y_min_scanlines = 16;
+                        break;
+                case 2:
+                        y_min_scanlines = 8;
+                        break;
+                case 4:
+                        y_min_scanlines = 4;
+                        break;
+                default:
+                        MISSING_CASE(cpp);
+                        return -EINVAL;
+                }
+        } else {
+                y_min_scanlines = 4;
+        }
+
+        plane_bytes_per_line = width * cpp;
+        if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
+            fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
+                plane_blocks_per_line =
+                      DIV_ROUND_UP(plane_bytes_per_line * y_min_scanlines, 512);
+                plane_blocks_per_line /= y_min_scanlines;
+        } else if (fb->modifier[0] == DRM_FORMAT_MOD_NONE) {
+                plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512)
+                                        + 1;
+        } else {
+                plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
+        }
+
         method1 = skl_wm_method1(plane_pixel_rate, cpp, latency);
         method2 = skl_wm_method2(plane_pixel_rate,
                                  cstate->base.adjusted_mode.crtc_htotal,
-                                 width,
-                                 cpp,
-                                 fb->modifier[0],
-                                 latency);
+                                 latency,
+                                 plane_blocks_per_line);
 
-        plane_bytes_per_line = width * cpp;
-        plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
+        y_tile_minimum = plane_blocks_per_line * y_min_scanlines;
+        if (apply_memory_bw_wa)
+                y_tile_minimum *= 2;
 
         if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
             fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
-                uint32_t min_scanlines = 4;
-                uint32_t y_tile_minimum;
-                if (drm_rotation_90_or_270(pstate->rotation)) {
-                        int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
-                                drm_format_plane_cpp(fb->pixel_format, 1) :
-                                drm_format_plane_cpp(fb->pixel_format, 0);
-
-                        switch (cpp) {
-                        case 1:
-                                min_scanlines = 16;
-                                break;
-                        case 2:
-                                min_scanlines = 8;
-                                break;
-                        case 8:
-                                WARN(1, "Unsupported pixel depth for rotation");
-                        }
-                }
-                y_tile_minimum = plane_blocks_per_line * min_scanlines;
                 selected_result = max(method2, y_tile_minimum);
         } else {
-                if ((ddb_allocation / plane_blocks_per_line) >= 1)
+                if ((cpp * cstate->base.adjusted_mode.crtc_htotal / 512 < 1) &&
+                    (plane_bytes_per_line / 512 < 1))
+                        selected_result = method2;
+                else if ((ddb_allocation / plane_blocks_per_line) >= 1)
                         selected_result = min(method1, method2);
                 else
                         selected_result = method1;
@@ -3605,10 +3665,12 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 
         if (level >= 1 && level <= 7) {
                 if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
-                    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)
-                        res_lines += 4;
-                else
+                    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
+                        res_blocks += y_tile_minimum;
+                        res_lines += y_min_scanlines;
+                } else {
                         res_blocks++;
+                }
         }
 
         if (res_blocks >= ddb_allocation || res_lines > 31) {
@@ -3642,67 +3704,52 @@ static int
 skl_compute_wm_level(const struct drm_i915_private *dev_priv,
                      struct skl_ddb_allocation *ddb,
                      struct intel_crtc_state *cstate,
+                     struct intel_plane *intel_plane,
                      int level,
                      struct skl_wm_level *result)
 {
         struct drm_atomic_state *state = cstate->base.state;
         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
-        struct drm_plane *plane;
-        struct intel_plane *intel_plane;
-        struct intel_plane_state *intel_pstate;
+        struct drm_plane *plane = &intel_plane->base;
+        struct intel_plane_state *intel_pstate = NULL;
         uint16_t ddb_blocks;
         enum pipe pipe = intel_crtc->pipe;
         int ret;
+        int i = skl_wm_plane_id(intel_plane);
+
+        if (state)
+                intel_pstate =
+                        intel_atomic_get_existing_plane_state(state,
+                                                              intel_plane);
 
         /*
-         * We'll only calculate watermarks for planes that are actually
-         * enabled, so make sure all other planes are set as disabled.
+         * Note: If we start supporting multiple pending atomic commits against
+         * the same planes/CRTC's in the future, plane->state will no longer be
+         * the correct pre-state to use for the calculations here and we'll
+         * need to change where we get the 'unchanged' plane data from.
+         *
+         * For now this is fine because we only allow one queued commit against
+         * a CRTC.  Even if the plane isn't modified by this transaction and we
+         * don't have a plane lock, we still have the CRTC's lock, so we know
+         * that no other transactions are racing with us to update it.
          */
-        memset(result, 0, sizeof(*result));
+        if (!intel_pstate)
+                intel_pstate = to_intel_plane_state(plane->state);
 
-        for_each_intel_plane_mask(&dev_priv->drm,
-                                  intel_plane,
-                                  cstate->base.plane_mask) {
-                int i = skl_wm_plane_id(intel_plane);
+        WARN_ON(!intel_pstate->base.fb);
 
-                plane = &intel_plane->base;
-                intel_pstate = NULL;
-                if (state)
-                        intel_pstate =
-                                intel_atomic_get_existing_plane_state(state,
-                                                                      intel_plane);
+        ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
 
-                /*
-                 * Note: If we start supporting multiple pending atomic commits
-                 * against the same planes/CRTC's in the future, plane->state
-                 * will no longer be the correct pre-state to use for the
-                 * calculations here and we'll need to change where we get the
-                 * 'unchanged' plane data from.
-                 *
-                 * For now this is fine because we only allow one queued commit
-                 * against a CRTC.  Even if the plane isn't modified by this
-                 * transaction and we don't have a plane lock, we still have
-                 * the CRTC's lock, so we know that no other transactions are
-                 * racing with us to update it.
-                 */
-                if (!intel_pstate)
-                        intel_pstate = to_intel_plane_state(plane->state);
-
-                WARN_ON(!intel_pstate->base.fb);
-
-                ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
-
-                ret = skl_compute_plane_wm(dev_priv,
-                                           cstate,
-                                           intel_pstate,
-                                           ddb_blocks,
-                                           level,
-                                           &result->plane_res_b[i],
-                                           &result->plane_res_l[i],
-                                           &result->plane_en[i]);
-                if (ret)
-                        return ret;
-        }
+        ret = skl_compute_plane_wm(dev_priv,
+                                   cstate,
+                                   intel_pstate,
+                                   ddb_blocks,
+                                   level,
+                                   &result->plane_res_b,
+                                   &result->plane_res_l,
+                                   &result->plane_en);
+        if (ret)
+                return ret;
 
         return 0;
 }
@@ -3710,32 +3757,28 @@ skl_compute_wm_level(const struct drm_i915_private *dev_priv,
 static uint32_t
 skl_compute_linetime_wm(struct intel_crtc_state *cstate)
 {
+        uint32_t pixel_rate;
+
         if (!cstate->base.active)
                 return 0;
 
-        if (WARN_ON(skl_pipe_pixel_rate(cstate) == 0))
+        pixel_rate = ilk_pipe_pixel_rate(cstate);
+
+        if (WARN_ON(pixel_rate == 0))
                 return 0;
 
         return DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal * 1000,
-                            skl_pipe_pixel_rate(cstate));
+                            pixel_rate);
 }
 
 static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
                                       struct skl_wm_level *trans_wm /* out */)
 {
-        struct drm_crtc *crtc = cstate->base.crtc;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_plane *intel_plane;
-
         if (!cstate->base.active)
                 return;
 
         /* Until we know more, just disable transition WMs */
-        for_each_intel_plane_on_crtc(crtc->dev, intel_crtc, intel_plane) {
-                int i = skl_wm_plane_id(intel_plane);
-
-                trans_wm->plane_en[i] = false;
-        }
+        trans_wm->plane_en = false;
 }
 
 static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
@@ -3744,77 +3787,34 @@ static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
 {
         struct drm_device *dev = cstate->base.crtc->dev;
         const struct drm_i915_private *dev_priv = to_i915(dev);
-        int level, max_level = ilk_wm_max_level(dev);
+        struct intel_plane *intel_plane;
+        struct skl_plane_wm *wm;
+        int level, max_level = ilk_wm_max_level(dev_priv);
         int ret;
 
-        for (level = 0; level <= max_level; level++) {
-                ret = skl_compute_wm_level(dev_priv, ddb, cstate,
-                                           level, &pipe_wm->wm[level]);
-                if (ret)
-                        return ret;
-        }
-        pipe_wm->linetime = skl_compute_linetime_wm(cstate);
-
-        skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
-
-        return 0;
-}
-
-static void skl_compute_wm_results(struct drm_device *dev,
-                                   struct skl_pipe_wm *p_wm,
-                                   struct skl_wm_values *r,
-                                   struct intel_crtc *intel_crtc)
-{
-        int level, max_level = ilk_wm_max_level(dev);
-        enum pipe pipe = intel_crtc->pipe;
-        uint32_t temp;
-        int i;
-
-        for (level = 0; level <= max_level; level++) {
-                for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-                        temp = 0;
-
-                        temp |= p_wm->wm[level].plane_res_l[i] <<
-                                        PLANE_WM_LINES_SHIFT;
-                        temp |= p_wm->wm[level].plane_res_b[i];
-                        if (p_wm->wm[level].plane_en[i])
-                                temp |= PLANE_WM_EN;
+        /*
+         * We'll only calculate watermarks for planes that are actually
+         * enabled, so make sure all other planes are set as disabled.
+         */
+        memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
 
-                        r->plane[pipe][i][level] = temp;
+        for_each_intel_plane_mask(&dev_priv->drm,
+                                  intel_plane,
+                                  cstate->base.plane_mask) {
+                wm = &pipe_wm->planes[skl_wm_plane_id(intel_plane)];
+
+                for (level = 0; level <= max_level; level++) {
+                        ret = skl_compute_wm_level(dev_priv, ddb, cstate,
+                                                   intel_plane, level,
+                                                   &wm->wm[level]);
+                        if (ret)
+                                return ret;
                 }
-
-                temp = 0;
-
-                temp |= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
-                temp |= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
-
-                if (p_wm->wm[level].plane_en[PLANE_CURSOR])
-                        temp |= PLANE_WM_EN;
-
-                r->plane[pipe][PLANE_CURSOR][level] = temp;
-
-        }
-
-        /* transition WMs */
-        for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-                temp = 0;
-                temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
-                temp |= p_wm->trans_wm.plane_res_b[i];
-                if (p_wm->trans_wm.plane_en[i])
-                        temp |= PLANE_WM_EN;
-
-                r->plane_trans[pipe][i] = temp;
+                skl_compute_transition_wm(cstate, &wm->trans_wm);
         }
+        pipe_wm->linetime = skl_compute_linetime_wm(cstate);
 
-        temp = 0;
-        temp |= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
-        temp |= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
-        if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
-                temp |= PLANE_WM_EN;
-
-        r->plane_trans[pipe][PLANE_CURSOR] = temp;
-
-        r->wm_linetime[pipe] = p_wm->linetime;
+        return 0;
 }
 
 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
@@ -3827,53 +3827,77 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
                 I915_WRITE(reg, 0);
 }
 
+static void skl_write_wm_level(struct drm_i915_private *dev_priv,
+                               i915_reg_t reg,
+                               const struct skl_wm_level *level)
+{
+        uint32_t val = 0;
+
+        if (level->plane_en) {
+                val |= PLANE_WM_EN;
+                val |= level->plane_res_b;
+                val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
+        }
+
+        I915_WRITE(reg, val);
+}
+
 void skl_write_plane_wm(struct intel_crtc *intel_crtc,
-                        const struct skl_wm_values *wm,
+                        const struct skl_plane_wm *wm,
+                        const struct skl_ddb_allocation *ddb,
                         int plane)
 {
         struct drm_crtc *crtc = &intel_crtc->base;
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
-        int level, max_level = ilk_wm_max_level(dev);
+        int level, max_level = ilk_wm_max_level(dev_priv);
         enum pipe pipe = intel_crtc->pipe;
 
         for (level = 0; level <= max_level; level++) {
-                I915_WRITE(PLANE_WM(pipe, plane, level),
-                           wm->plane[pipe][plane][level]);
+                skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane, level),
+                                   &wm->wm[level]);
         }
-        I915_WRITE(PLANE_WM_TRANS(pipe, plane), wm->plane_trans[pipe][plane]);
+        skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane),
+                           &wm->trans_wm);
 
         skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane),
-                            &wm->ddb.plane[pipe][plane]);
+                            &ddb->plane[pipe][plane]);
         skl_ddb_entry_write(dev_priv, PLANE_NV12_BUF_CFG(pipe, plane),
-                            &wm->ddb.y_plane[pipe][plane]);
+                            &ddb->y_plane[pipe][plane]);
 }
 
 void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
-                         const struct skl_wm_values *wm)
+                         const struct skl_plane_wm *wm,
+                         const struct skl_ddb_allocation *ddb)
 {
         struct drm_crtc *crtc = &intel_crtc->base;
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
-        int level, max_level = ilk_wm_max_level(dev);
+        int level, max_level = ilk_wm_max_level(dev_priv);
         enum pipe pipe = intel_crtc->pipe;
 
         for (level = 0; level <= max_level; level++) {
-                I915_WRITE(CUR_WM(pipe, level),
-                           wm->plane[pipe][PLANE_CURSOR][level]);
+                skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
+                                   &wm->wm[level]);
         }
-        I915_WRITE(CUR_WM_TRANS(pipe), wm->plane_trans[pipe][PLANE_CURSOR]);
+        skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
 
         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
-                            &wm->ddb.plane[pipe][PLANE_CURSOR]);
+                            &ddb->plane[pipe][PLANE_CURSOR]);
 }
 
-bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old,
-                               const struct skl_ddb_allocation *new,
-                               enum pipe pipe)
+bool skl_wm_level_equals(const struct skl_wm_level *l1,
+                         const struct skl_wm_level *l2)
 {
-        return new->pipe[pipe].start == old->pipe[pipe].start &&
-               new->pipe[pipe].end == old->pipe[pipe].end;
+        if (l1->plane_en != l2->plane_en)
+                return false;
+
+        /* If both planes aren't enabled, the rest shouldn't matter */
+        if (!l1->plane_en)
+                return true;
+
+        return (l1->plane_res_l == l2->plane_res_l &&
+                l1->plane_res_b == l2->plane_res_b);
 }
 
 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
@@ -3883,22 +3907,22 @@ static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
 }
 
 bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state,
-                                 const struct skl_ddb_allocation *old,
-                                 const struct skl_ddb_allocation *new,
-                                 enum pipe pipe)
+                                 struct intel_crtc *intel_crtc)
 {
-        struct drm_device *dev = state->dev;
-        struct intel_crtc *intel_crtc;
-        enum pipe otherp;
+        struct drm_crtc *other_crtc;
+        struct drm_crtc_state *other_cstate;
+        struct intel_crtc *other_intel_crtc;
+        const struct skl_ddb_entry *ddb =
+                &to_intel_crtc_state(intel_crtc->base.state)->wm.skl.ddb;
+        int i;
 
-        for_each_intel_crtc(dev, intel_crtc) {
-                otherp = intel_crtc->pipe;
+        for_each_crtc_in_state(state, other_crtc, other_cstate, i) {
+                other_intel_crtc = to_intel_crtc(other_crtc);
 
-                if (otherp == pipe)
+                if (other_intel_crtc == intel_crtc)
                         continue;
 
-                if (skl_ddb_entries_overlap(&new->pipe[pipe],
-                                            &old->pipe[otherp]))
+                if (skl_ddb_entries_overlap(ddb, &other_intel_crtc->hw_ddb))
                         return true;
         }
 
@@ -3940,6 +3964,41 @@ pipes_modified(struct drm_atomic_state *state)
 }
 
 static int
+skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
+{
+        struct drm_atomic_state *state = cstate->base.state;
+        struct drm_device *dev = state->dev;
+        struct drm_crtc *crtc = cstate->base.crtc;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+        struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
+        struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
+        struct drm_plane_state *plane_state;
+        struct drm_plane *plane;
+        enum pipe pipe = intel_crtc->pipe;
+        int id;
+
+        WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc));
+
+        drm_for_each_plane_mask(plane, dev, crtc->state->plane_mask) {
+                id = skl_wm_plane_id(to_intel_plane(plane));
+
+                if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][id],
+                                        &new_ddb->plane[pipe][id]) &&
+                    skl_ddb_entry_equal(&cur_ddb->y_plane[pipe][id],
+                                        &new_ddb->y_plane[pipe][id]))
+                        continue;
+
+                plane_state = drm_atomic_get_plane_state(state, plane);
+                if (IS_ERR(plane_state))
+                        return PTR_ERR(plane_state);
+        }
+
+        return 0;
+}
+
+static int
 skl_compute_ddb(struct drm_atomic_state *state)
 {
         struct drm_device *dev = state->dev;
@@ -3992,6 +4051,12 @@ skl_compute_ddb(struct drm_atomic_state *state)
                 intel_state->wm_results.dirty_pipes = ~0;
         }
 
+        /*
+         * We're not recomputing for the pipes not included in the commit, so
+         * make sure we start with the current state.
+         */
+        memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
+
         for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
                 struct intel_crtc_state *cstate;
 
@@ -4003,7 +4068,7 @@ skl_compute_ddb(struct drm_atomic_state *state)
                 if (ret)
                         return ret;
 
-                ret = drm_atomic_add_affected_planes(state, &intel_crtc->base);
+                ret = skl_ddb_add_affected_planes(cstate);
                 if (ret)
                         return ret;
         }
@@ -4016,19 +4081,64 @@ skl_copy_wm_for_pipe(struct skl_wm_values *dst,
                      struct skl_wm_values *src,
                      enum pipe pipe)
 {
-        dst->wm_linetime[pipe] = src->wm_linetime[pipe];
-        memcpy(dst->plane[pipe], src->plane[pipe],
-               sizeof(dst->plane[pipe]));
-        memcpy(dst->plane_trans[pipe], src->plane_trans[pipe],
-               sizeof(dst->plane_trans[pipe]));
-
-        dst->ddb.pipe[pipe] = src->ddb.pipe[pipe];
         memcpy(dst->ddb.y_plane[pipe], src->ddb.y_plane[pipe],
                sizeof(dst->ddb.y_plane[pipe]));
         memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
                sizeof(dst->ddb.plane[pipe]));
 }
 
+static void
+skl_print_wm_changes(const struct drm_atomic_state *state)
+{
+        const struct drm_device *dev = state->dev;
+        const struct drm_i915_private *dev_priv = to_i915(dev);
+        const struct intel_atomic_state *intel_state =
+                to_intel_atomic_state(state);
+        const struct drm_crtc *crtc;
+        const struct drm_crtc_state *cstate;
+        const struct drm_plane *plane;
+        const struct intel_plane *intel_plane;
+        const struct drm_plane_state *pstate;
+        const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
+        const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
+        enum pipe pipe;
+        int id;
+        int i, j;
+
+        for_each_crtc_in_state(state, crtc, cstate, i) {
+                pipe = to_intel_crtc(crtc)->pipe;
+
+                for_each_plane_in_state(state, plane, pstate, j) {
+                        const struct skl_ddb_entry *old, *new;
+
+                        intel_plane = to_intel_plane(plane);
+                        id = skl_wm_plane_id(intel_plane);
+                        old = &old_ddb->plane[pipe][id];
+                        new = &new_ddb->plane[pipe][id];
+
+                        if (intel_plane->pipe != pipe)
+                                continue;
+
+                        if (skl_ddb_entry_equal(old, new))
+                                continue;
+
+                        if (id != PLANE_CURSOR) {
+                                DRM_DEBUG_ATOMIC("[PLANE:%d:plane %d%c] ddb (%d - %d) -> (%d - %d)\n",
+                                                 plane->base.id, id + 1,
+                                                 pipe_name(pipe),
+                                                 old->start, old->end,
+                                                 new->start, new->end);
+                        } else {
+                                DRM_DEBUG_ATOMIC("[PLANE:%d:cursor %c] ddb (%d - %d) -> (%d - %d)\n",
+                                                 plane->base.id,
+                                                 pipe_name(pipe),
+                                                 old->start, old->end,
+                                                 new->start, new->end);
+                        }
+                }
+        }
+}
+
 static int
 skl_compute_wm(struct drm_atomic_state *state)
 {
@@ -4071,7 +4181,6 @@ skl_compute_wm(struct drm_atomic_state *state)
          * no suitable watermark values can be found.
          */
         for_each_crtc_in_state(state, crtc, cstate, i) {
-                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
                 struct intel_crtc_state *intel_cstate =
                         to_intel_crtc_state(cstate);
 
@@ -4089,9 +4198,10 @@ skl_compute_wm(struct drm_atomic_state *state)
                         continue;
 
                 intel_cstate->update_wm_pre = true;
-                skl_compute_wm_results(crtc->dev, pipe_wm, results, intel_crtc);
         }
 
+        skl_print_wm_changes(state);
+
         return 0;
 }
 
@@ -4123,13 +4233,17 @@ static void skl_update_wm(struct drm_crtc *crtc)
                 int plane;
 
                 for (plane = 0; plane < intel_num_planes(intel_crtc); plane++)
-                        skl_write_plane_wm(intel_crtc, results, plane);
+                        skl_write_plane_wm(intel_crtc, &pipe_wm->planes[plane],
+                                           &results->ddb, plane);
 
-                skl_write_cursor_wm(intel_crtc, results);
+                skl_write_cursor_wm(intel_crtc, &pipe_wm->planes[PLANE_CURSOR],
+                                    &results->ddb);
         }
 
         skl_copy_wm_for_pipe(hw_vals, results, pipe);
 
+        intel_crtc->hw_ddb = cstate->wm.skl.ddb;
+
         mutex_unlock(&dev_priv->wm.wm_mutex);
 }
 
@@ -4208,114 +4322,77 @@ static void ilk_optimize_watermarks(struct intel_crtc_state *cstate)
         mutex_unlock(&dev_priv->wm.wm_mutex);
 }
 
-static void skl_pipe_wm_active_state(uint32_t val,
-                                     struct skl_pipe_wm *active,
-                                     bool is_transwm,
-                                     bool is_cursor,
-                                     int i,
-                                     int level)
+static inline void skl_wm_level_from_reg_val(uint32_t val,
+                                             struct skl_wm_level *level)
 {
-        bool is_enabled = (val & PLANE_WM_EN) != 0;
-
-        if (!is_transwm) {
-                if (!is_cursor) {
-                        active->wm[level].plane_en[i] = is_enabled;
-                        active->wm[level].plane_res_b[i] =
-                                        val & PLANE_WM_BLOCKS_MASK;
-                        active->wm[level].plane_res_l[i] =
-                                        (val >> PLANE_WM_LINES_SHIFT) &
-                                                PLANE_WM_LINES_MASK;
-                } else {
-                        active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
-                        active->wm[level].plane_res_b[PLANE_CURSOR] =
-                                        val & PLANE_WM_BLOCKS_MASK;
-                        active->wm[level].plane_res_l[PLANE_CURSOR] =
-                                        (val >> PLANE_WM_LINES_SHIFT) &
-                                                PLANE_WM_LINES_MASK;
-                }
-        } else {
-                if (!is_cursor) {
-                        active->trans_wm.plane_en[i] = is_enabled;
-                        active->trans_wm.plane_res_b[i] =
-                                        val & PLANE_WM_BLOCKS_MASK;
-                        active->trans_wm.plane_res_l[i] =
-                                        (val >> PLANE_WM_LINES_SHIFT) &
-                                                PLANE_WM_LINES_MASK;
-                } else {
-                        active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
-                        active->trans_wm.plane_res_b[PLANE_CURSOR] =
-                                        val & PLANE_WM_BLOCKS_MASK;
-                        active->trans_wm.plane_res_l[PLANE_CURSOR] =
-                                        (val >> PLANE_WM_LINES_SHIFT) &
-                                                PLANE_WM_LINES_MASK;
-                }
-        }
+        level->plane_en = val & PLANE_WM_EN;
+        level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
+        level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
+                PLANE_WM_LINES_MASK;
 }
 
-static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
+void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
+                              struct skl_pipe_wm *out)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
-        struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
-        struct skl_pipe_wm *active = &cstate->wm.skl.optimal;
+        struct intel_plane *intel_plane;
+        struct skl_plane_wm *wm;
         enum pipe pipe = intel_crtc->pipe;
-        int level, i, max_level;
-        uint32_t temp;
-
-        max_level = ilk_wm_max_level(dev);
-
-        hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
-
-        for (level = 0; level <= max_level; level++) {
-                for (i = 0; i < intel_num_planes(intel_crtc); i++)
-                        hw->plane[pipe][i][level] =
-                                        I915_READ(PLANE_WM(pipe, i, level));
-                hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
-        }
-
-        for (i = 0; i < intel_num_planes(intel_crtc); i++)
-                hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
-        hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
+        int level, id, max_level;
+        uint32_t val;
 
-        if (!intel_crtc->active)
-                return;
+        max_level = ilk_wm_max_level(dev_priv);
 
-        hw->dirty_pipes |= drm_crtc_mask(crtc);
+        for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+                id = skl_wm_plane_id(intel_plane);
+                wm = &out->planes[id];
 
-        active->linetime = hw->wm_linetime[pipe];
+                for (level = 0; level <= max_level; level++) {
+                        if (id != PLANE_CURSOR)
+                                val = I915_READ(PLANE_WM(pipe, id, level));
+                        else
+                                val = I915_READ(CUR_WM(pipe, level));
 
-        for (level = 0; level <= max_level; level++) {
-                for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-                        temp = hw->plane[pipe][i][level];
-                        skl_pipe_wm_active_state(temp, active, false,
-                                                false, i, level);
+                        skl_wm_level_from_reg_val(val, &wm->wm[level]);
                 }
-                temp = hw->plane[pipe][PLANE_CURSOR][level];
-                skl_pipe_wm_active_state(temp, active, false, true, i, level);
-        }
 
-        for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-                temp = hw->plane_trans[pipe][i];
-                skl_pipe_wm_active_state(temp, active, true, false, i, 0);
+                if (id != PLANE_CURSOR)
+                        val = I915_READ(PLANE_WM_TRANS(pipe, id));
+                else
+                        val = I915_READ(CUR_WM_TRANS(pipe));
+
+                skl_wm_level_from_reg_val(val, &wm->trans_wm);
         }
 
-        temp = hw->plane_trans[pipe][PLANE_CURSOR];
-        skl_pipe_wm_active_state(temp, active, true, true, i, 0);
+        if (!intel_crtc->active)
+                return;
 
-        intel_crtc->wm.active.skl = *active;
+        out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
 }
 
 void skl_wm_get_hw_state(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
+        struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
         struct drm_crtc *crtc;
+        struct intel_crtc *intel_crtc;
+        struct intel_crtc_state *cstate;
 
         skl_ddb_get_hw_state(dev_priv, ddb);
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-                skl_pipe_wm_get_hw_state(crtc);
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                intel_crtc = to_intel_crtc(crtc);
+                cstate = to_intel_crtc_state(crtc->state);
+
+                skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
+
+                if (intel_crtc->active) {
+                        hw->dirty_pipes |= drm_crtc_mask(crtc);
+                        intel_crtc->wm.active.skl = cstate->wm.skl.optimal;
+                }
+        }
 
         if (dev_priv->active_crtcs) {
                 /* Fully recompute DDB on first atomic commit */
@@ -4342,7 +4419,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
         };
 
         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
 
         memset(active, 0, sizeof(*active));
@@ -4364,7 +4441,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
                 active->linetime = hw->wm_linetime[pipe];
         } else {
-                int level, max_level = ilk_wm_max_level(dev);
+                int level, max_level = ilk_wm_max_level(dev_priv);
 
                 /*
                  * For inactive pipes, all watermark levels
@@ -4550,10 +4627,10 @@ void ilk_wm_get_hw_state(struct drm_device *dev)
                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
         }
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
-        else if (IS_IVYBRIDGE(dev))
+        else if (IS_IVYBRIDGE(dev_priv))
                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
 
@@ -5297,6 +5374,7 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         uint32_t rc6_mask = 0;
 
         /* 1a: Software RC state - RC0 */
@@ -5318,7 +5396,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 
         if (HAS_GUC(dev_priv))
@@ -5334,9 +5412,8 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
         if (intel_enable_rc6() & INTEL_RC6_ENABLE)
                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
         DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
-        /* WaRsUseTimeoutMode */
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
-            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
+        /* WaRsUseTimeoutMode:bxt */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
                            GEN7_RC_CTL_TO_MODE |
@@ -5364,6 +5441,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         uint32_t rc6_mask = 0;
 
         /* 1a: Software RC state - RC0 */
@@ -5380,7 +5458,7 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
         I915_WRITE(GEN6_RC_SLEEP, 0);
         if (IS_BROADWELL(dev_priv))
@@ -5440,6 +5518,7 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         u32 rc6vids, rc6_mask = 0;
         u32 gtfifodbg;
         int rc6_mode;
@@ -5473,7 +5552,7 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 
         I915_WRITE(GEN6_RC_SLEEP, 0);
@@ -5510,10 +5589,6 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
 
-        ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
-        if (ret)
-                DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
-
         reset_rps(dev_priv, gen6_set_rps);
 
         rc6vids = 0;
@@ -5922,6 +5997,7 @@ static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         u32 gtfifodbg, val, rc6_mode = 0, pcbr;
 
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
@@ -5948,7 +6024,7 @@ static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
         I915_WRITE(GEN6_RC_SLEEP, 0);
 
@@ -6010,6 +6086,7 @@ static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         u32 gtfifodbg, val, rc6_mode = 0;
 
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
@@ -6049,7 +6126,7 @@ static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
 
-        for_each_engine(engine, dev_priv)
+        for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 
         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
@@ -6732,7 +6809,7 @@ static void __intel_autoenable_gt_powersave(struct work_struct *work)
         if (READ_ONCE(dev_priv->rps.enabled))
                 goto out;
 
-        rcs = &dev_priv->engine[RCS];
+        rcs = dev_priv->engine[RCS];
         if (rcs->last_context)
                 goto out;
 
@@ -6869,7 +6946,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
          * The bit 22 of 0x42004
          * The bit 7,8,9 of 0x42020.
          */
-        if (IS_IRONLAKE_M(dev)) {
+        if (IS_IRONLAKE_M(dev_priv)) {
                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
                            I915_READ(ILK_DISPLAY_CHICKEN1) |
@@ -7071,7 +7148,7 @@ static void lpt_init_clock_gating(struct drm_device *dev)
          * TODO: this bit should only be enabled when really needed, then
          * disabled when not needed anymore in order to save power.
          */
-        if (HAS_PCH_LPT_LP(dev))
+        if (HAS_PCH_LPT_LP(dev_priv))
                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
                            I915_READ(SOUTH_DSPCLK_GATE_D) |
                            PCH_LP_PARTITION_LEVEL_DISABLE);
@@ -7086,7 +7163,7 @@ static void lpt_suspend_hw(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (HAS_PCH_LPT_LP(dev)) {
+        if (HAS_PCH_LPT_LP(dev_priv)) {
                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
 
                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
@@ -7279,7 +7356,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
         /* WaDisablePSDDualDispatchEnable:ivb */
-        if (IS_IVB_GT1(dev))
+        if (IS_IVB_GT1(dev_priv))
                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
 
@@ -7295,7 +7372,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
                         GEN7_WA_FOR_GEN7_L3_CONTROL);
         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
                    GEN7_WA_L3_CHICKEN_MODE);
-        if (IS_IVB_GT1(dev))
+        if (IS_IVB_GT1(dev_priv))
                 I915_WRITE(GEN7_ROW_CHICKEN2,
                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
         else {
@@ -7352,7 +7429,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
         snpcr |= GEN6_MBC_SNPCR_MED;
         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
 
-        if (!HAS_PCH_NOP(dev))
+        if (!HAS_PCH_NOP(dev_priv))
                 cpt_init_clock_gating(dev);
 
         gen6_check_mch_setup(dev);
@@ -7489,7 +7566,7 @@ static void g4x_init_clock_gating(struct drm_device *dev)
         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
                 OVRUNIT_CLOCK_GATE_DISABLE |
                 OVCUNIT_CLOCK_GATE_DISABLE;
-        if (IS_GM45(dev))
+        if (IS_GM45(dev_priv))
                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
 
@@ -7595,7 +7672,7 @@ void intel_init_clock_gating(struct drm_device *dev)
 
 void intel_suspend_hw(struct drm_device *dev)
 {
-        if (HAS_PCH_LPT(dev))
+        if (HAS_PCH_LPT(to_i915(dev)))
                 lpt_suspend_hw(dev);
 }
 
@@ -7663,7 +7740,7 @@ void intel_init_pm(struct drm_device *dev)
         /* For cxsr */
         if (IS_PINEVIEW(dev))
                 i915_pineview_get_mem_freq(dev);
-        else if (IS_GEN5(dev))
+        else if (IS_GEN5(dev_priv))
                 i915_ironlake_get_mem_freq(dev);
 
         /* For FIFO watermark updates */
@@ -7671,12 +7748,12 @@ void intel_init_pm(struct drm_device *dev)
                 skl_setup_wm_latency(dev);
                 dev_priv->display.update_wm = skl_update_wm;
                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
                 ilk_setup_wm_latency(dev);
 
-                if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
+                if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
-                    (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
+                    (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
                         dev_priv->display.compute_intermediate_wm =
@@ -7689,14 +7766,14 @@ void intel_init_pm(struct drm_device *dev)
                         DRM_DEBUG_KMS("Failed to read display plane latency. "
                                       "Disable CxSR\n");
                 }
-        } else if (IS_CHERRYVIEW(dev)) {
+        } else if (IS_CHERRYVIEW(dev_priv)) {
                 vlv_setup_wm_latency(dev);
                 dev_priv->display.update_wm = vlv_update_wm;
-        } else if (IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv)) {
                 vlv_setup_wm_latency(dev);
                 dev_priv->display.update_wm = vlv_update_wm;
         } else if (IS_PINEVIEW(dev)) {
-                if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
+                if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
                                             dev_priv->is_ddr3,
                                             dev_priv->fsb_freq,
                                             dev_priv->mem_freq)) {
@@ -7710,14 +7787,14 @@ void intel_init_pm(struct drm_device *dev)
                         dev_priv->display.update_wm = NULL;
                 } else
                         dev_priv->display.update_wm = pineview_update_wm;
-        } else if (IS_G4X(dev)) {
+        } else if (IS_G4X(dev_priv)) {
                 dev_priv->display.update_wm = g4x_update_wm;
-        } else if (IS_GEN4(dev)) {
+        } else if (IS_GEN4(dev_priv)) {
                 dev_priv->display.update_wm = i965_update_wm;
-        } else if (IS_GEN3(dev)) {
+        } else if (IS_GEN3(dev_priv)) {
                 dev_priv->display.update_wm = i9xx_update_wm;
                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
-        } else if (IS_GEN2(dev)) {
+        } else if (IS_GEN2(dev_priv)) {
                 if (INTEL_INFO(dev)->num_pipes == 1) {
                         dev_priv->display.update_wm = i845_update_wm;
                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 108ba1e5d658..271a3e29ff23 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -268,7 +268,7 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp)
         val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
         val |= idle_frames << EDP_PSR_IDLE_FRAME_SHIFT;
 
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev_priv))
                 val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
 
         if (dev_priv->psr.link_standby)
@@ -344,7 +344,7 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp)
          * ones. Since by Display design transcoder EDP is tied to port A
          * we can safely escape based on the port A.
          */
-        if (HAS_DDI(dev) && dig_port->port != PORT_A) {
+        if (HAS_DDI(dev_priv) && dig_port->port != PORT_A) {
                 DRM_DEBUG_KMS("PSR condition failed: Port not supported\n");
                 return false;
         }
@@ -354,20 +354,20 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp)
                 return false;
         }
 
-        if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+        if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
             !dev_priv->psr.link_standby) {
                 DRM_ERROR("PSR condition failed: Link off requested but not supported on this platform\n");
                 return false;
         }
 
-        if (IS_HASWELL(dev) &&
+        if (IS_HASWELL(dev_priv) &&
             I915_READ(HSW_STEREO_3D_CTL(intel_crtc->config->cpu_transcoder)) &
                       S3D_ENABLE) {
                 DRM_DEBUG_KMS("PSR condition failed: Stereo 3D is Enabled\n");
                 return false;
         }
 
-        if (IS_HASWELL(dev) &&
+        if (IS_HASWELL(dev_priv) &&
             adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
                 DRM_DEBUG_KMS("PSR condition failed: Interlaced is Enabled\n");
                 return false;
@@ -402,7 +402,7 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
         lockdep_assert_held(&dev_priv->psr.lock);
 
         /* Enable/Re-enable PSR on the host */
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev_priv))
                 /* On HSW+ after we enable PSR on source it will activate it
                  * as soon as it match configure idle_frame count. So
                  * we just actually enable it here on activation time.
@@ -448,7 +448,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 
         dev_priv->psr.busy_frontbuffer_bits = 0;
 
-        if (HAS_DDI(dev)) {
+        if (HAS_DDI(dev_priv)) {
                 hsw_psr_setup_vsc(intel_dp);
 
                 if (dev_priv->psr.psr2_support) {
@@ -580,7 +580,7 @@ void intel_psr_disable(struct intel_dp *intel_dp)
         }
 
         /* Disable PSR on Source */
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev_priv))
                 hsw_psr_disable(intel_dp);
         else
                 vlv_psr_disable(intel_dp);
@@ -827,17 +827,17 @@ void intel_psr_init(struct drm_device *dev)
 
         /* Per platform default */
         if (i915.enable_psr == -1) {
-                if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+                if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                         i915.enable_psr = 1;
                 else
                         i915.enable_psr = 0;
         }
 
         /* Set link_standby x link_off defaults */
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 /* HSW and BDW require workarounds that we don't implement. */
                 dev_priv->psr.link_standby = false;
-        else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+        else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 /* On VLV and CHV only standby mode is supported. */
                 dev_priv->psr.link_standby = true;
         else
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 7a74750076c5..32786ba199b9 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -405,22 +405,6 @@ gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
         return gen8_emit_pipe_control(req, flags, scratch_addr);
 }
 
-u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
-{
-        struct drm_i915_private *dev_priv = engine->i915;
-        u64 acthd;
-
-        if (INTEL_GEN(dev_priv) >= 8)
-                acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
-                                         RING_ACTHD_UDW(engine->mmio_base));
-        else if (INTEL_GEN(dev_priv) >= 4)
-                acthd = I915_READ(RING_ACTHD(engine->mmio_base));
-        else
-                acthd = I915_READ(ACTHD);
-
-        return acthd;
-}
-
 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
 {
         struct drm_i915_private *dev_priv = engine->i915;
@@ -564,7 +548,7 @@ static int init_ring_common(struct intel_engine_cs *engine)
         else
                 intel_ring_setup_status_page(engine);
 
-        intel_engine_reset_irq(engine);
+        intel_engine_reset_breadcrumbs(engine);
 
         /* Enforce ordering by reading HEAD register back */
         I915_READ_HEAD(engine);
@@ -585,9 +569,7 @@ static int init_ring_common(struct intel_engine_cs *engine)
         I915_WRITE_TAIL(engine, ring->tail);
         (void)I915_READ_TAIL(engine);
 
-        I915_WRITE_CTL(engine,
-                        ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
-                        | RING_VALID);
+        I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
 
         /* If the head is still not zero, the ring is dead */
         if (intel_wait_for_register_fw(dev_priv, RING_CTL(engine->mmio_base),
@@ -851,15 +833,13 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
                           GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 
-        /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
-            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
+        /* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
                                   GEN9_DG_MIRROR_FIX_ENABLE);
 
-        /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
-            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
+        /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
                                   GEN9_RHWO_OPTIMIZATION_DISABLE);
                 /*
@@ -869,10 +849,8 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
                  */
         }
 
-        /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl */
         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl */
         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
-                          GEN9_ENABLE_YV12_BUGFIX |
                           GEN9_ENABLE_GPGPU_PREEMPTION);
 
         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl */
@@ -884,9 +862,8 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 
-        /* WaDisableMaskBasedCammingInRCC:skl,bxt */
-        if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_C0) ||
-            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
+        /* WaDisableMaskBasedCammingInRCC:bxt */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
                                   PIXEL_MASK_CAMMING_DISABLE);
 
@@ -1003,47 +980,12 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
          * until D0 which is the default case so this is equivalent to
          * !WaDisablePerCtxtPreemptionGranularityControl:skl
          */
-        if (IS_SKL_REVID(dev_priv, SKL_REVID_E0, REVID_FOREVER)) {
-                I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
-                           _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
-        }
-
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0)) {
-                /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
-                I915_WRITE(FF_SLICE_CS_CHICKEN2,
-                           _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
-        }
-
-        /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
-         * involving this register should also be added to WA batch as required.
-         */
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0))
-                /* WaDisableLSQCROPERFforOCL:skl */
-                I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
-                           GEN8_LQSC_RO_PERF_DIS);
+        I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
+                   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
 
         /* WaEnableGapsTsvCreditFix:skl */
-        if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, REVID_FOREVER)) {
-                I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
-                                           GEN9_GAPS_TSV_CREDIT_DISABLE));
-        }
-
-        /* WaDisablePowerCompilerClockGating:skl */
-        if (IS_SKL_REVID(dev_priv, SKL_REVID_B0, SKL_REVID_B0))
-                WA_SET_BIT_MASKED(HIZ_CHICKEN,
-                                  BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
-
-        /* WaBarrierPerformanceFixDisable:skl */
-        if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_D0))
-                WA_SET_BIT_MASKED(HDC_CHICKEN0,
-                                  HDC_FENCE_DEST_SLM_DISABLE |
-                                  HDC_BARRIER_PERFORMANCE_DISABLE);
-
-        /* WaDisableSbeCacheDispatchPortSharing:skl */
-        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0))
-                WA_SET_BIT_MASKED(
-                        GEN7_HALF_SLICE_CHICKEN1,
-                        GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
+        I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
+                                   GEN9_GAPS_TSV_CREDIT_DISABLE));
 
         /* WaDisableGafsUnitClkGating:skl */
         WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
@@ -1284,7 +1226,7 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *req)
         if (ret)
                 return ret;
 
-        for_each_engine_id(waiter, dev_priv, id) {
+        for_each_engine(waiter, dev_priv, id) {
                 u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
                         continue;
@@ -1321,7 +1263,7 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *req)
         if (ret)
                 return ret;
 
-        for_each_engine_id(waiter, dev_priv, id) {
+        for_each_engine(waiter, dev_priv, id) {
                 u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
                         continue;
@@ -1348,6 +1290,7 @@ static int gen6_signal(struct drm_i915_gem_request *req)
         struct intel_ring *ring = req->ring;
         struct drm_i915_private *dev_priv = req->i915;
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
         int ret, num_rings;
 
         num_rings = INTEL_INFO(dev_priv)->num_rings;
@@ -1355,7 +1298,7 @@ static int gen6_signal(struct drm_i915_gem_request *req)
         if (ret)
                 return ret;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 i915_reg_t mbox_reg;
 
                 if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
@@ -1989,6 +1932,7 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)
         struct i915_vma *vma;
 
         GEM_BUG_ON(!is_power_of_2(size));
+        GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
 
         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
         if (!ring)
@@ -2146,9 +2090,6 @@ void intel_engine_cleanup(struct intel_engine_cs *engine)
 {
         struct drm_i915_private *dev_priv;
 
-        if (!intel_engine_initialized(engine))
-                return;
-
         dev_priv = engine->i915;
 
         if (engine->buffer) {
@@ -2175,13 +2116,16 @@ void intel_engine_cleanup(struct intel_engine_cs *engine)
         intel_ring_context_unpin(dev_priv->kernel_context, engine);
 
         engine->i915 = NULL;
+        dev_priv->engine[engine->id] = NULL;
+        kfree(engine);
 }
 
 void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
+        enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv) {
+        for_each_engine(engine, dev_priv, id) {
                 engine->buffer->head = engine->buffer->tail;
                 engine->buffer->last_retired_head = -1;
         }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 7f64d611159b..32b2e6332ccf 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -73,13 +73,40 @@ enum intel_engine_hangcheck_action {
 
 #define HANGCHECK_SCORE_RING_HUNG 31
 
+#define I915_MAX_SLICES 3
+#define I915_MAX_SUBSLICES 3
+
+#define instdone_slice_mask(dev_priv__) \
+        (INTEL_GEN(dev_priv__) == 7 ? \
+         1 : INTEL_INFO(dev_priv__)->sseu.slice_mask)
+
+#define instdone_subslice_mask(dev_priv__) \
+        (INTEL_GEN(dev_priv__) == 7 ? \
+         1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask)
+
+#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
+        for ((slice__) = 0, (subslice__) = 0; \
+             (slice__) < I915_MAX_SLICES; \
+             (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
+               (slice__) += ((subslice__) == 0)) \
+                for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
+                            (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
+
+struct intel_instdone {
+        u32 instdone;
+        /* The following exist only in the RCS engine */
+        u32 slice_common;
+        u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
+        u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
+};
+
 struct intel_engine_hangcheck {
         u64 acthd;
         u32 seqno;
         int score;
         enum intel_engine_hangcheck_action action;
         int deadlock;
-        u32 instdone[I915_NUM_INSTDONE_REG];
+        struct intel_instdone instdone;
 };
 
 struct intel_ring {
@@ -328,6 +355,7 @@ struct intel_engine_cs {
          * inspecting request list.
          */
         u32 last_submitted_seqno;
+        u32 last_pending_seqno;
 
         /* An RCU guarded pointer to the last request. No reference is
          * held to the request, users must carefully acquire a reference to
@@ -367,12 +395,6 @@ struct intel_engine_cs {
         u32 (*get_cmd_length_mask)(u32 cmd_header);
 };
 
-static inline bool
-intel_engine_initialized(const struct intel_engine_cs *engine)
-{
-        return engine->i915 != NULL;
-}
-
 static inline unsigned
 intel_engine_flag(const struct intel_engine_cs *engine)
 {
@@ -393,7 +415,7 @@ intel_engine_sync_index(struct intel_engine_cs *engine,
          * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
          */
 
-        idx = (other - engine) - 1;
+        idx = (other->id - engine->id) - 1;
         if (idx < 0)
                 idx += I915_NUM_ENGINES;
 
@@ -492,7 +514,6 @@ int __intel_ring_space(int head, int tail, int size);
 void intel_ring_update_space(struct intel_ring *ring);
 
 void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno);
-void intel_engine_reset_irq(struct intel_engine_cs *engine);
 
 void intel_engine_setup_common(struct intel_engine_cs *engine);
 int intel_engine_init_common(struct intel_engine_cs *engine);
@@ -514,6 +535,8 @@ int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
 int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
 
 u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
+u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine);
+
 static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
 {
         return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
@@ -521,6 +544,9 @@ static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
 
 int init_workarounds_ring(struct intel_engine_cs *engine);
 
+void intel_engine_get_instdone(struct intel_engine_cs *engine,
+                               struct intel_instdone *instdone);
+
 /*
  * Arbitrary size for largest possible 'add request' sequence. The code paths
  * are complex and variable. Empirical measurement shows that the worst case
@@ -584,6 +610,7 @@ static inline bool intel_engine_wakeup(const struct intel_engine_cs *engine)
         return wakeup;
 }
 
+void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
 unsigned int intel_kick_waiters(struct drm_i915_private *i915);
 unsigned int intel_kick_signalers(struct drm_i915_private *i915);
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 6c11168facd6..ee56a8756c07 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -288,7 +288,6 @@ void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 {
         struct pci_dev *pdev = dev_priv->drm.pdev;
-        struct drm_device *dev = &dev_priv->drm;
 
         /*
          * After we re-enable the power well, if we touch VGA register 0x3d5
@@ -304,7 +303,7 @@ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
         outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
         vga_put(pdev, VGA_RSRC_LEGACY_IO);
 
-        if (IS_BROADWELL(dev))
+        if (IS_BROADWELL(dev_priv))
                 gen8_irq_power_well_post_enable(dev_priv,
                                                 1 << PIPE_C | 1 << PIPE_B);
 }
@@ -2590,20 +2589,19 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
  */
 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
 {
-        struct drm_device *dev = &dev_priv->drm;
         struct i915_power_domains *power_domains = &dev_priv->power_domains;
 
         power_domains->initializing = true;
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 skl_display_core_init(dev_priv, resume);
-        } else if (IS_BROXTON(dev)) {
+        } else if (IS_BROXTON(dev_priv)) {
                 bxt_display_core_init(dev_priv, resume);
-        } else if (IS_CHERRYVIEW(dev)) {
+        } else if (IS_CHERRYVIEW(dev_priv)) {
                 mutex_lock(&power_domains->lock);
                 chv_phy_control_init(dev_priv);
                 mutex_unlock(&power_domains->lock);
-        } else if (IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv)) {
                 mutex_lock(&power_domains->lock);
                 vlv_cmnlane_wa(dev_priv);
                 mutex_unlock(&power_domains->lock);
@@ -2758,7 +2756,6 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 {
         struct pci_dev *pdev = dev_priv->drm.pdev;
-        struct drm_device *dev = &dev_priv->drm;
         struct device *kdev = &pdev->dev;
 
         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
@@ -2770,7 +2767,7 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
          * so the driver's own RPM reference tracking asserts also work on
          * platforms without RPM support.
          */
-        if (!HAS_RUNTIME_PM(dev)) {
+        if (!HAS_RUNTIME_PM(dev_priv)) {
                 pm_runtime_dont_use_autosuspend(kdev);
                 pm_runtime_get_sync(kdev);
         } else {
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index c551024d4871..49fb95d03d74 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -251,7 +251,7 @@ static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val)
                  * HW workaround, need to write this twice for issue
                  * that may result in first write getting masked.
                  */
-                if (HAS_PCH_IBX(dev)) {
+                if (HAS_PCH_IBX(dev_priv)) {
                         I915_WRITE(intel_sdvo->sdvo_reg, val);
                         POSTING_READ(intel_sdvo->sdvo_reg);
                 }
@@ -307,7 +307,7 @@ static bool intel_sdvo_read_byte(struct intel_sdvo *intel_sdvo, u8 addr, u8 *ch)
 static const struct _sdvo_cmd_name {
         u8 cmd;
         const char *name;
-} sdvo_cmd_names[] = {
+} __attribute__ ((packed)) sdvo_cmd_names[] = {
         SDVO_CMD_NAME_ENTRY(SDVO_CMD_RESET),
         SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_DEVICE_CAPS),
         SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_FIRMWARE_REV),
@@ -1133,7 +1133,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
         DRM_DEBUG_KMS("forcing bpc to 8 for SDVO\n");
         pipe_config->pipe_bpp = 8*3;
 
-        if (HAS_PCH_SPLIT(encoder->base.dev))
+        if (HAS_PCH_SPLIT(to_i915(encoder->base.dev)))
                 pipe_config->has_pch_encoder = true;
 
         /* We need to construct preferred input timings based on our
@@ -1273,7 +1273,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
                 /* The real mode polarity is set by the SDVO commands, using
                  * struct intel_sdvo_dtd. */
                 sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH;
-                if (!HAS_PCH_SPLIT(dev) && crtc_state->limited_color_range)
+                if (!HAS_PCH_SPLIT(dev_priv) && crtc_state->limited_color_range)
                         sdvox |= HDMI_COLOR_RANGE_16_235;
                 if (INTEL_INFO(dev)->gen < 5)
                         sdvox |= SDVO_BORDER_ENABLE;
@@ -1286,7 +1286,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
                 sdvox |= (9 << 19) | SDVO_BORDER_ENABLE;
         }
 
-        if (INTEL_PCH_TYPE(dev) >= PCH_CPT)
+        if (INTEL_PCH_TYPE(dev_priv) >= PCH_CPT)
                 sdvox |= SDVO_PIPE_SEL_CPT(crtc->pipe);
         else
                 sdvox |= SDVO_PIPE_SEL(crtc->pipe);
@@ -1296,7 +1296,8 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 
         if (INTEL_INFO(dev)->gen >= 4) {
                 /* done in crtc_mode_set as the dpll_md reg must be written early */
-        } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+        } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
+                   IS_G33(dev_priv)) {
                 /* done in crtc_mode_set as it lives inside the dpll register */
         } else {
                 sdvox |= (crtc_state->pixel_multiplier - 1)
@@ -1339,7 +1340,7 @@ static bool intel_sdvo_get_hw_state(struct intel_encoder *encoder,
         if (!(tmp & SDVO_ENABLE) && (active_outputs == 0))
                 return false;
 
-        if (HAS_PCH_CPT(dev))
+        if (HAS_PCH_CPT(dev_priv))
                 *pipe = PORT_TO_PIPE_CPT(tmp);
         else
                 *pipe = PORT_TO_PIPE(tmp);
@@ -1389,7 +1390,7 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
          * encoder->get_config we so already have a valid pixel multplier on all
          * other platfroms.
          */
-        if (IS_I915G(dev) || IS_I915GM(dev)) {
+        if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
                 pipe_config->pixel_multiplier =
                         ((sdvox & SDVO_PORT_MULTIPLY_MASK)
                          >> SDVO_PORT_MULTIPLY_SHIFT) + 1;
@@ -1595,15 +1596,15 @@ static bool intel_sdvo_get_capabilities(struct intel_sdvo *intel_sdvo, struct in
 
 static uint16_t intel_sdvo_get_hotplug_support(struct intel_sdvo *intel_sdvo)
 {
-        struct drm_device *dev = intel_sdvo->base.base.dev;
+        struct drm_i915_private *dev_priv = to_i915(intel_sdvo->base.base.dev);
         uint16_t hotplug;
 
-        if (!I915_HAS_HOTPLUG(dev))
+        if (!I915_HAS_HOTPLUG(dev_priv))
                 return 0;
 
         /* HW Erratum: SDVO Hotplug is broken on all i945G chips, there's noise
          * on the line. */
-        if (IS_I945G(dev) || IS_I945GM(dev))
+        if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
                 return 0;
 
         if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_HOT_PLUG_SUPPORT,
@@ -2981,6 +2982,7 @@ bool intel_sdvo_init(struct drm_device *dev,
         /* encoder type will be decided later */
         intel_encoder = &intel_sdvo->base;
         intel_encoder->type = INTEL_OUTPUT_SDVO;
+        intel_encoder->port = port;
         drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0,
                          "SDVO %c", port_name(port));
 
@@ -2996,7 +2998,7 @@ bool intel_sdvo_init(struct drm_device *dev,
         }
 
         intel_encoder->compute_config = intel_sdvo_compute_config;
-        if (HAS_PCH_SPLIT(dev)) {
+        if (HAS_PCH_SPLIT(dev_priv)) {
                 intel_encoder->disable = pch_disable_sdvo;
                 intel_encoder->post_disable = pch_post_disable_sdvo;
         } else {
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 3ea6419e18b9..43d0350856e7 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -208,6 +208,8 @@ skl_update_plane(struct drm_plane *drm_plane,
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         const int pipe = intel_plane->pipe;
         const int plane = intel_plane->plane + 1;
+        const struct skl_plane_wm *p_wm =
+                &crtc_state->wm.skl.optimal.planes[plane];
         u32 plane_ctl;
         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
         u32 surf_addr = plane_state->main.offset;
@@ -232,7 +234,7 @@ skl_update_plane(struct drm_plane *drm_plane,
         plane_ctl |= skl_plane_ctl_rotation(rotation);
 
         if (wm->dirty_pipes & drm_crtc_mask(crtc))
-                skl_write_plane_wm(intel_crtc, wm, plane);
+                skl_write_plane_wm(intel_crtc, p_wm, &wm->ddb, plane);
 
         if (key->flags) {
                 I915_WRITE(PLANE_KEYVAL(pipe, plane), key->min_value);
@@ -289,6 +291,7 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
         struct drm_device *dev = dplane->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(dplane);
+        struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
         const int pipe = intel_plane->pipe;
         const int plane = intel_plane->plane + 1;
 
@@ -298,7 +301,8 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
          */
         if (!dplane->state->visible)
                 skl_write_plane_wm(to_intel_crtc(crtc),
-                                   &dev_priv->wm.skl_results, plane);
+                                   &cstate->wm.skl.optimal.planes[plane],
+                                   &dev_priv->wm.skl_results.ddb, plane);
 
         I915_WRITE(PLANE_CTL(pipe, plane), 0);
 
@@ -450,7 +454,7 @@ vlv_update_plane(struct drm_plane *dplane,
         if (key->flags & I915_SET_COLORKEY_SOURCE)
                 sprctl |= SP_SOURCE_KEY;
 
-        if (IS_CHERRYVIEW(dev) && pipe == PIPE_B)
+        if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B)
                 chv_update_csc(intel_plane, fb->pixel_format);
 
         I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]);
@@ -542,12 +546,12 @@ ivb_update_plane(struct drm_plane *plane,
         if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
                 sprctl |= SPRITE_TILED;
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 sprctl &= ~SPRITE_TRICKLE_FEED_DISABLE;
         else
                 sprctl |= SPRITE_TRICKLE_FEED_DISABLE;
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 sprctl |= SPRITE_PIPE_CSC_ENABLE;
 
         /* Sizes are 0 based */
@@ -566,7 +570,7 @@ ivb_update_plane(struct drm_plane *plane,
                 sprctl |= SPRITE_ROTATE_180;
 
                 /* HSW and BDW does this automagically in hardware */
-                if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
+                if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
                         x += src_w;
                         y += src_h;
                 }
@@ -590,7 +594,7 @@ ivb_update_plane(struct drm_plane *plane,
 
         /* HSW consolidates SPRTILEOFF and SPRLINOFF into a single SPROFFSET
          * register */
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 I915_WRITE(SPROFFSET(pipe), (y << 16) | x);
         else if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
                 I915_WRITE(SPRTILEOFF(pipe), (y << 16) | x);
@@ -680,7 +684,7 @@ ilk_update_plane(struct drm_plane *plane,
         if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
                 dvscntr |= DVS_TILED;
 
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev_priv))
                 dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */
 
         /* Sizes are 0 based */
@@ -753,7 +757,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
                          struct intel_crtc_state *crtc_state,
                          struct intel_plane_state *state)
 {
-        struct drm_device *dev = plane->dev;
+        struct drm_i915_private *dev_priv = to_i915(plane->dev);
         struct drm_crtc *crtc = state->base.crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_plane *intel_plane = to_intel_plane(plane);
@@ -797,7 +801,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
         }
 
         /* setup can_scale, min_scale, max_scale */
-        if (INTEL_INFO(dev)->gen >= 9) {
+        if (INTEL_GEN(dev_priv) >= 9) {
                 /* use scaler when colorkey is not required */
                 if (state->ckey.flags == I915_SET_COLORKEY_NONE) {
                         can_scale = 1;
@@ -913,7 +917,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 
                 width_bytes = ((src_x * cpp) & 63) + src_w * cpp;
 
-                if (INTEL_INFO(dev)->gen < 9 && (src_w > 2048 || src_h > 2048 ||
+                if (INTEL_GEN(dev_priv) < 9 && (src_w > 2048 || src_h > 2048 ||
                     width_bytes > 4096 || fb->pitches[0] > 4096)) {
                         DRM_DEBUG_KMS("Source dimensions exceed hardware limits\n");
                         return -EINVAL;
@@ -932,7 +936,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
         dst->y1 = crtc_y;
         dst->y2 = crtc_y + crtc_h;
 
-        if (INTEL_GEN(dev) >= 9) {
+        if (INTEL_GEN(dev_priv) >= 9) {
                 ret = skl_check_plane_surface(state);
                 if (ret)
                         return ret;
@@ -944,6 +948,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
                               struct drm_file *file_priv)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_intel_sprite_colorkey *set = data;
         struct drm_plane *plane;
         struct drm_plane_state *plane_state;
@@ -955,7 +960,7 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
         if ((set->flags & (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE)) == (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
                 return -EINVAL;
 
-        if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+        if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
             set->flags & I915_SET_COLORKEY_DESTINATION)
                 return -EINVAL;
 
@@ -1040,6 +1045,7 @@ static uint32_t skl_plane_formats[] = {
 int
 intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = NULL;
         struct intel_plane_state *state = NULL;
         unsigned long possible_crtcs;
@@ -1072,7 +1078,7 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
                 intel_plane->update_plane = ilk_update_plane;
                 intel_plane->disable_plane = ilk_disable_plane;
 
-                if (IS_GEN6(dev)) {
+                if (IS_GEN6(dev_priv)) {
                         plane_formats = snb_plane_formats;
                         num_plane_formats = ARRAY_SIZE(snb_plane_formats);
                 } else {
@@ -1083,7 +1089,7 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 
         case 7:
         case 8:
-                if (IS_IVYBRIDGE(dev)) {
+                if (IS_IVYBRIDGE(dev_priv)) {
                         intel_plane->can_scale = true;
                         intel_plane->max_downscale = 2;
                 } else {
@@ -1091,7 +1097,7 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
                         intel_plane->max_downscale = 1;
                 }
 
-                if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+                if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                         intel_plane->update_plane = vlv_update_plane;
                         intel_plane->disable_plane = vlv_disable_plane;
 
@@ -1120,7 +1126,7 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
                 goto fail;
         }
 
-        if (INTEL_GEN(dev) >= 9) {
+        if (INTEL_GEN(dev_priv) >= 9) {
                 supported_rotations =
                         DRM_ROTATE_0 | DRM_ROTATE_90 |
                         DRM_ROTATE_180 | DRM_ROTATE_270;
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index d960e4866595..7118fb55f57f 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -86,7 +86,8 @@ struct intel_tv {
 };
 
 struct video_levels {
-        int blank, black, burst;
+        u16 blank, black;
+        u8 burst;
 };
 
 struct color_conversion {
@@ -339,34 +340,43 @@ static const struct video_levels component_levels = {
 
 struct tv_mode {
         const char *name;
-        int clock;
-        int refresh; /* in millihertz (for precision) */
+
+        u32 clock;
+        u16 refresh; /* in millihertz (for precision) */
         u32 oversample;
-        int hsync_end, hblank_start, hblank_end, htotal;
-        bool progressive, trilevel_sync, component_only;
-        int vsync_start_f1, vsync_start_f2, vsync_len;
-        bool veq_ena;
-        int veq_start_f1, veq_start_f2, veq_len;
-        int vi_end_f1, vi_end_f2, nbr_end;
-        bool burst_ena;
-        int hburst_start, hburst_len;
-        int vburst_start_f1, vburst_end_f1;
-        int vburst_start_f2, vburst_end_f2;
-        int vburst_start_f3, vburst_end_f3;
-        int vburst_start_f4, vburst_end_f4;
+        u8 hsync_end;
+        u16 hblank_start, hblank_end, htotal;
+        bool progressive : 1, trilevel_sync : 1, component_only : 1;
+        u8 vsync_start_f1, vsync_start_f2, vsync_len;
+        bool veq_ena : 1;
+        u8 veq_start_f1, veq_start_f2, veq_len;
+        u8 vi_end_f1, vi_end_f2;
+        u16 nbr_end;
+        bool burst_ena : 1;
+        u8 hburst_start, hburst_len;
+        u8 vburst_start_f1;
+        u16 vburst_end_f1;
+        u8 vburst_start_f2;
+        u16 vburst_end_f2;
+        u8 vburst_start_f3;
+        u16 vburst_end_f3;
+        u8 vburst_start_f4;
+        u16 vburst_end_f4;
         /*
          * subcarrier programming
          */
-        int dda2_size, dda3_size, dda1_inc, dda2_inc, dda3_inc;
+        u16 dda2_size, dda3_size;
+        u8 dda1_inc;
+        u16 dda2_inc, dda3_inc;
         u32 sc_reset;
-        bool pal_burst;
+        bool pal_burst : 1;
         /*
          * blank/black levels
          */
         const struct video_levels *composite_levels, *svideo_levels;
         const struct color_conversion *composite_color, *svideo_color;
         const u32 *filter_table;
-        int max_srcw;
+        u16 max_srcw;
 };
 
 
@@ -1095,7 +1105,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
                 tv_mode->dda3_inc << TV_SCDDA3_INC_SHIFT;
 
         /* Enable two fixes for the chips that need them. */
-        if (IS_I915GM(dev))
+        if (IS_I915GM(dev_priv))
                 tv_ctl |= TV_ENC_C0_FIX | TV_ENC_SDP_FIX;
 
         set_tv_mode_timings(dev_priv, tv_mode, burst_ena);
@@ -1220,7 +1230,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
          * The TV sense state should be cleared to zero on cantiga platform. Otherwise
          * the TV is misdetected. This is hardware requirement.
          */
-        if (IS_GM45(dev))
+        if (IS_GM45(dev_priv))
                 tv_dac &= ~(TVDAC_STATE_CHG_EN | TVDAC_A_SENSE_CTL |
                             TVDAC_B_SENSE_CTL | TVDAC_C_SENSE_CTL);
 
@@ -1610,7 +1620,9 @@ intel_tv_init(struct drm_device *dev)
         intel_connector->get_hw_state = intel_connector_get_hw_state;
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
+
         intel_encoder->type = INTEL_OUTPUT_TVOUT;
+        intel_encoder->port = PORT_NONE;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
         intel_encoder->cloneable = 0;
         intel_encoder->base.possible_crtcs = ((1 << 0) | (1 << 1));
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index a9b6c936aadd..e2b188dcf908 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -231,19 +231,21 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
 {
         struct intel_uncore_forcewake_domain *domain =
                container_of(timer, struct intel_uncore_forcewake_domain, timer);
+        struct drm_i915_private *dev_priv = domain->i915;
         unsigned long irqflags;
 
-        assert_rpm_device_not_suspended(domain->i915);
+        assert_rpm_device_not_suspended(dev_priv);
 
-        spin_lock_irqsave(&domain->i915->uncore.lock, irqflags);
+        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
         if (WARN_ON(domain->wake_count == 0))
                 domain->wake_count++;
 
-        if (--domain->wake_count == 0)
-                domain->i915->uncore.funcs.force_wake_put(domain->i915,
-                                                          1 << domain->id);
+        if (--domain->wake_count == 0) {
+                dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
+                dev_priv->uncore.fw_domains_active &= ~domain->mask;
+        }
 
-        spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags);
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 
         return HRTIMER_NORESTART;
 }
@@ -254,7 +256,7 @@ void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
         unsigned long irqflags;
         struct intel_uncore_forcewake_domain *domain;
         int retry_count = 100;
-        enum forcewake_domains fw = 0, active_domains;
+        enum forcewake_domains fw, active_domains;
 
         /* Hold uncore.lock across reset to prevent any register access
          * with forcewake not set correctly. Wait until all pending
@@ -291,10 +293,7 @@ void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 
         WARN_ON(active_domains);
 
-        for_each_fw_domain(domain, dev_priv)
-                if (domain->wake_count)
-                        fw |= domain->mask;
-
+        fw = dev_priv->uncore.fw_domains_active;
         if (fw)
                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
 
@@ -443,9 +442,6 @@ static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 {
         struct intel_uncore_forcewake_domain *domain;
 
-        if (!dev_priv->uncore.funcs.force_wake_get)
-                return;
-
         fw_domains &= dev_priv->uncore.fw_domains;
 
         for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
@@ -453,8 +449,10 @@ static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
                         fw_domains &= ~domain->mask;
         }
 
-        if (fw_domains)
+        if (fw_domains) {
                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+                dev_priv->uncore.fw_domains_active |= fw_domains;
+        }
 }
 
 /**
@@ -509,9 +507,6 @@ static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 {
         struct intel_uncore_forcewake_domain *domain;
 
-        if (!dev_priv->uncore.funcs.force_wake_put)
-                return;
-
         fw_domains &= dev_priv->uncore.fw_domains;
 
         for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
@@ -567,13 +562,10 @@ void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
 
 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
 {
-        struct intel_uncore_forcewake_domain *domain;
-
         if (!dev_priv->uncore.funcs.force_wake_get)
                 return;
 
-        for_each_fw_domain(domain, dev_priv)
-                WARN_ON(domain->wake_count);
+        WARN_ON(dev_priv->uncore.fw_domains_active);
 }
 
 /* We give fast paths for the really cool registers */
@@ -589,49 +581,146 @@ void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
         __fwd; \
 })
 
-#define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end))
+static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
+{
+        if (offset < entry->start)
+                return -1;
+        else if (offset > entry->end)
+                return 1;
+        else
+                return 0;
+}
+
+/* Copied and "macroized" from lib/bsearch.c */
+#define BSEARCH(key, base, num, cmp) ({                                 \
+        unsigned int start__ = 0, end__ = (num);                        \
+        typeof(base) result__ = NULL;                                   \
+        while (start__ < end__) {                                       \
+                unsigned int mid__ = start__ + (end__ - start__) / 2;   \
+                int ret__ = (cmp)((key), (base) + mid__);               \
+                if (ret__ < 0) {                                        \
+                        end__ = mid__;                                  \
+                } else if (ret__ > 0) {                                 \
+                        start__ = mid__ + 1;                            \
+                } else {                                                \
+                        result__ = (base) + mid__;                      \
+                        break;                                          \
+                }                                                       \
+        }                                                               \
+        result__;                                                       \
+})
+
+static enum forcewake_domains
+find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
+{
+        const struct intel_forcewake_range *entry;
+
+        entry = BSEARCH(offset,
+                        dev_priv->uncore.fw_domains_table,
+                        dev_priv->uncore.fw_domains_table_entries,
+                        fw_range_cmp);
 
-#define FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg) \
-        (REG_RANGE((reg), 0x2000, 0x4000) || \
-         REG_RANGE((reg), 0x5000, 0x8000) || \
-         REG_RANGE((reg), 0xB000, 0x12000) || \
-         REG_RANGE((reg), 0x2E000, 0x30000))
+        return entry ? entry->domains : 0;
+}
+
+static void
+intel_fw_table_check(struct drm_i915_private *dev_priv)
+{
+        const struct intel_forcewake_range *ranges;
+        unsigned int num_ranges;
+        s32 prev;
+        unsigned int i;
+
+        if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+                return;
+
+        ranges = dev_priv->uncore.fw_domains_table;
+        if (!ranges)
+                return;
+
+        num_ranges = dev_priv->uncore.fw_domains_table_entries;
+
+        for (i = 0, prev = -1; i < num_ranges; i++, ranges++) {
+                WARN_ON_ONCE(prev >= (s32)ranges->start);
+                prev = ranges->start;
+                WARN_ON_ONCE(prev >= (s32)ranges->end);
+                prev = ranges->end;
+        }
+}
 
-#define FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg) \
-        (REG_RANGE((reg), 0x12000, 0x14000) || \
-         REG_RANGE((reg), 0x22000, 0x24000) || \
-         REG_RANGE((reg), 0x30000, 0x40000))
+#define GEN_FW_RANGE(s, e, d) \
+        { .start = (s), .end = (e), .domains = (d) }
 
-#define __vlv_reg_read_fw_domains(offset) \
+#define HAS_FWTABLE(dev_priv) \
+        (IS_GEN9(dev_priv) || \
+         IS_CHERRYVIEW(dev_priv) || \
+         IS_VALLEYVIEW(dev_priv))
+
+/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
+static const struct intel_forcewake_range __vlv_fw_ranges[] = {
+        GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
+};
+
+#define __fwtable_reg_read_fw_domains(offset) \
 ({ \
         enum forcewake_domains __fwd = 0; \
-        if (!NEEDS_FORCE_WAKE(offset)) \
-                __fwd = 0; \
-        else if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER; \
-        else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_MEDIA; \
+        if (NEEDS_FORCE_WAKE((offset))) \
+                __fwd = find_fw_domain(dev_priv, offset); \
         __fwd; \
 })
 
+/* *Must* be sorted by offset! See intel_shadow_table_check(). */
 static const i915_reg_t gen8_shadowed_regs[] = {
-        GEN6_RPNSWREQ,
-        GEN6_RC_VIDEO_FREQ,
-        RING_TAIL(RENDER_RING_BASE),
-        RING_TAIL(GEN6_BSD_RING_BASE),
-        RING_TAIL(VEBOX_RING_BASE),
-        RING_TAIL(BLT_RING_BASE),
+        RING_TAIL(RENDER_RING_BASE),    /* 0x2000 (base) */
+        GEN6_RPNSWREQ,                  /* 0xA008 */
+        GEN6_RC_VIDEO_FREQ,             /* 0xA00C */
+        RING_TAIL(GEN6_BSD_RING_BASE),  /* 0x12000 (base) */
+        RING_TAIL(VEBOX_RING_BASE),     /* 0x1a000 (base) */
+        RING_TAIL(BLT_RING_BASE),       /* 0x22000 (base) */
         /* TODO: Other registers are not yet used */
 };
 
+static void intel_shadow_table_check(void)
+{
+        const i915_reg_t *reg = gen8_shadowed_regs;
+        s32 prev;
+        u32 offset;
+        unsigned int i;
+
+        if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+                return;
+
+        for (i = 0, prev = -1; i < ARRAY_SIZE(gen8_shadowed_regs); i++, reg++) {
+                offset = i915_mmio_reg_offset(*reg);
+                WARN_ON_ONCE(prev >= (s32)offset);
+                prev = offset;
+        }
+}
+
+static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
+{
+        u32 offset = i915_mmio_reg_offset(*reg);
+
+        if (key < offset)
+                return -1;
+        else if (key > offset)
+                return 1;
+        else
+                return 0;
+}
+
 static bool is_gen8_shadowed(u32 offset)
 {
-        int i;
-        for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
-                if (offset == gen8_shadowed_regs[i].reg)
-                        return true;
+        const i915_reg_t *regs = gen8_shadowed_regs;
 
-        return false;
+        return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
+                       mmio_reg_cmp);
 }
 
 #define __gen8_reg_write_fw_domains(offset) \
@@ -644,143 +733,70 @@ static bool is_gen8_shadowed(u32 offset)
         __fwd; \
 })
 
-#define FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg) \
-        (REG_RANGE((reg), 0x2000, 0x4000) || \
-         REG_RANGE((reg), 0x5200, 0x8000) || \
-         REG_RANGE((reg), 0x8300, 0x8500) || \
-         REG_RANGE((reg), 0xB000, 0xB480) || \
-         REG_RANGE((reg), 0xE000, 0xE800))
-
-#define FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg) \
-        (REG_RANGE((reg), 0x8800, 0x8900) || \
-         REG_RANGE((reg), 0xD000, 0xD800) || \
-         REG_RANGE((reg), 0x12000, 0x14000) || \
-         REG_RANGE((reg), 0x1A000, 0x1C000) || \
-         REG_RANGE((reg), 0x1E800, 0x1EA00) || \
-         REG_RANGE((reg), 0x30000, 0x38000))
-
-#define FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg) \
-        (REG_RANGE((reg), 0x4000, 0x5000) || \
-         REG_RANGE((reg), 0x8000, 0x8300) || \
-         REG_RANGE((reg), 0x8500, 0x8600) || \
-         REG_RANGE((reg), 0x9000, 0xB000) || \
-         REG_RANGE((reg), 0xF000, 0x10000))
-
-#define __chv_reg_read_fw_domains(offset) \
-({ \
-        enum forcewake_domains __fwd = 0; \
-        if (!NEEDS_FORCE_WAKE(offset)) \
-                __fwd = 0; \
-        else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER; \
-        else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_MEDIA; \
-        else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
-        __fwd; \
-})
+/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
+static const struct intel_forcewake_range __chv_fw_ranges[] = {
+        GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
+};
 
-#define __chv_reg_write_fw_domains(offset) \
+#define __fwtable_reg_write_fw_domains(offset) \
 ({ \
         enum forcewake_domains __fwd = 0; \
-        if (!NEEDS_FORCE_WAKE(offset) || is_gen8_shadowed(offset)) \
-                __fwd = 0; \
-        else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER; \
-        else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_MEDIA; \
-        else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
-        __fwd; \
-})
-
-#define FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg) \
-        REG_RANGE((reg), 0xB00,  0x2000)
-
-#define FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg) \
-        (REG_RANGE((reg), 0x2000, 0x2700) || \
-         REG_RANGE((reg), 0x3000, 0x4000) || \
-         REG_RANGE((reg), 0x5200, 0x8000) || \
-         REG_RANGE((reg), 0x8140, 0x8160) || \
-         REG_RANGE((reg), 0x8300, 0x8500) || \
-         REG_RANGE((reg), 0x8C00, 0x8D00) || \
-         REG_RANGE((reg), 0xB000, 0xB480) || \
-         REG_RANGE((reg), 0xE000, 0xE900) || \
-         REG_RANGE((reg), 0x24400, 0x24800))
-
-#define FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg) \
-        (REG_RANGE((reg), 0x8130, 0x8140) || \
-         REG_RANGE((reg), 0x8800, 0x8A00) || \
-         REG_RANGE((reg), 0xD000, 0xD800) || \
-         REG_RANGE((reg), 0x12000, 0x14000) || \
-         REG_RANGE((reg), 0x1A000, 0x1EA00) || \
-         REG_RANGE((reg), 0x30000, 0x40000))
-
-#define FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg) \
-        REG_RANGE((reg), 0x9400, 0x9800)
-
-#define FORCEWAKE_GEN9_BLITTER_RANGE_OFFSET(reg) \
-        ((reg) < 0x40000 && \
-         !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg) && \
-         !FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg) && \
-         !FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg) && \
-         !FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg))
-
-#define SKL_NEEDS_FORCE_WAKE(reg) \
-        ((reg) < 0x40000 && !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg))
-
-#define __gen9_reg_read_fw_domains(offset) \
-({ \
-        enum forcewake_domains __fwd; \
-        if (!SKL_NEEDS_FORCE_WAKE(offset)) \
-                __fwd = 0; \
-        else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER; \
-        else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_MEDIA; \
-        else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
-        else \
-                __fwd = FORCEWAKE_BLITTER; \
+        if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
+                __fwd = find_fw_domain(dev_priv, offset); \
         __fwd; \
 })
 
-static const i915_reg_t gen9_shadowed_regs[] = {
-        RING_TAIL(RENDER_RING_BASE),
-        RING_TAIL(GEN6_BSD_RING_BASE),
-        RING_TAIL(VEBOX_RING_BASE),
-        RING_TAIL(BLT_RING_BASE),
-        GEN6_RPNSWREQ,
-        GEN6_RC_VIDEO_FREQ,
-        /* TODO: Other registers are not yet used */
+/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
+static const struct intel_forcewake_range __gen9_fw_ranges[] = {
+        GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
+        GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0xb480, 0xbfff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
+        GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
+        GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
+        GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 };
 
-static bool is_gen9_shadowed(u32 offset)
-{
-        int i;
-        for (i = 0; i < ARRAY_SIZE(gen9_shadowed_regs); i++)
-                if (offset == gen9_shadowed_regs[i].reg)
-                        return true;
-
-        return false;
-}
-
-#define __gen9_reg_write_fw_domains(offset) \
-({ \
-        enum forcewake_domains __fwd; \
-        if (!SKL_NEEDS_FORCE_WAKE(offset) || is_gen9_shadowed(offset)) \
-                __fwd = 0; \
-        else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER; \
-        else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_MEDIA; \
-        else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \
-                __fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
-        else \
-                __fwd = FORCEWAKE_BLITTER; \
-        __fwd; \
-})
-
 static void
 ilk_dummy_write(struct drm_i915_private *dev_priv)
 {
@@ -796,10 +812,9 @@ __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
                       const bool read,
                       const bool before)
 {
-        if (WARN(check_for_unclaimed_mmio(dev_priv),
-                 "Unclaimed register detected %s %s register 0x%x\n",
-                 before ? "before" : "after",
-                 read ? "reading" : "writing to",
+        if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
+                 "Unclaimed %s register 0x%x\n",
+                 read ? "read from" : "write to",
                  i915_mmio_reg_offset(reg)))
                 i915.mmio_debug--; /* Only report the first N failures */
 }
@@ -870,26 +885,30 @@ __gen2_read(64)
         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
         return val
 
-static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
-                                     enum forcewake_domains fw_domains)
+static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
+                                        enum forcewake_domains fw_domains)
 {
         struct intel_uncore_forcewake_domain *domain;
 
+        for_each_fw_domain_masked(domain, fw_domains, dev_priv)
+                fw_domain_arm_timer(domain);
+
+        dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+        dev_priv->uncore.fw_domains_active |= fw_domains;
+}
+
+static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
+                                     enum forcewake_domains fw_domains)
+{
         if (WARN_ON(!fw_domains))
                 return;
 
-        /* Ideally GCC would be constant-fold and eliminate this loop */
-        for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
-                if (domain->wake_count) {
-                        fw_domains &= ~domain->mask;
-                        continue;
-                }
-
-                fw_domain_arm_timer(domain);
-        }
+        /* Turn on all requested but inactive supported forcewake domains. */
+        fw_domains &= dev_priv->uncore.fw_domains;
+        fw_domains &= ~dev_priv->uncore.fw_domains_active;
 
         if (fw_domains)
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+                ___force_wake_auto(dev_priv, fw_domains);
 }
 
 #define __gen6_read(x) \
@@ -904,62 +923,28 @@ gen6_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
         GEN6_READ_FOOTER; \
 }
 
-#define __vlv_read(x) \
-static u##x \
-vlv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
-        enum forcewake_domains fw_engine; \
-        GEN6_READ_HEADER(x); \
-        fw_engine = __vlv_reg_read_fw_domains(offset); \
-        if (fw_engine) \
-                __force_wake_auto(dev_priv, fw_engine); \
-        val = __raw_i915_read##x(dev_priv, reg); \
-        GEN6_READ_FOOTER; \
-}
-
-#define __chv_read(x) \
+#define __fwtable_read(x) \
 static u##x \
-chv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+fwtable_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
         enum forcewake_domains fw_engine; \
         GEN6_READ_HEADER(x); \
-        fw_engine = __chv_reg_read_fw_domains(offset); \
+        fw_engine = __fwtable_reg_read_fw_domains(offset); \
         if (fw_engine) \
                 __force_wake_auto(dev_priv, fw_engine); \
         val = __raw_i915_read##x(dev_priv, reg); \
         GEN6_READ_FOOTER; \
 }
 
-#define __gen9_read(x) \
-static u##x \
-gen9_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
-        enum forcewake_domains fw_engine; \
-        GEN6_READ_HEADER(x); \
-        fw_engine = __gen9_reg_read_fw_domains(offset); \
-        if (fw_engine) \
-                __force_wake_auto(dev_priv, fw_engine); \
-        val = __raw_i915_read##x(dev_priv, reg); \
-        GEN6_READ_FOOTER; \
-}
-
-__gen9_read(8)
-__gen9_read(16)
-__gen9_read(32)
-__gen9_read(64)
-__chv_read(8)
-__chv_read(16)
-__chv_read(32)
-__chv_read(64)
-__vlv_read(8)
-__vlv_read(16)
-__vlv_read(32)
-__vlv_read(64)
+__fwtable_read(8)
+__fwtable_read(16)
+__fwtable_read(32)
+__fwtable_read(64)
 __gen6_read(8)
 __gen6_read(16)
 __gen6_read(32)
 __gen6_read(64)
 
-#undef __gen9_read
-#undef __chv_read
-#undef __vlv_read
+#undef __fwtable_read
 #undef __gen6_read
 #undef GEN6_READ_FOOTER
 #undef GEN6_READ_HEADER
@@ -1055,21 +1040,6 @@ gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool
         GEN6_WRITE_FOOTER; \
 }
 
-#define __hsw_write(x) \
-static void \
-hsw_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
-        u32 __fifo_ret = 0; \
-        GEN6_WRITE_HEADER; \
-        if (NEEDS_FORCE_WAKE(offset)) { \
-                __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
-        } \
-        __raw_i915_write##x(dev_priv, reg, val); \
-        if (unlikely(__fifo_ret)) { \
-                gen6_gt_check_fifodbg(dev_priv); \
-        } \
-        GEN6_WRITE_FOOTER; \
-}
-
 #define __gen8_write(x) \
 static void \
 gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
@@ -1082,51 +1052,30 @@ gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool
         GEN6_WRITE_FOOTER; \
 }
 
-#define __chv_write(x) \
-static void \
-chv_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
-        enum forcewake_domains fw_engine; \
-        GEN6_WRITE_HEADER; \
-        fw_engine = __chv_reg_write_fw_domains(offset); \
-        if (fw_engine) \
-                __force_wake_auto(dev_priv, fw_engine); \
-        __raw_i915_write##x(dev_priv, reg, val); \
-        GEN6_WRITE_FOOTER; \
-}
-
-#define __gen9_write(x) \
+#define __fwtable_write(x) \
 static void \
-gen9_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, \
-                bool trace) { \
+fwtable_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
         enum forcewake_domains fw_engine; \
         GEN6_WRITE_HEADER; \
-        fw_engine = __gen9_reg_write_fw_domains(offset); \
+        fw_engine = __fwtable_reg_write_fw_domains(offset); \
         if (fw_engine) \
                 __force_wake_auto(dev_priv, fw_engine); \
         __raw_i915_write##x(dev_priv, reg, val); \
         GEN6_WRITE_FOOTER; \
 }
 
-__gen9_write(8)
-__gen9_write(16)
-__gen9_write(32)
-__chv_write(8)
-__chv_write(16)
-__chv_write(32)
+__fwtable_write(8)
+__fwtable_write(16)
+__fwtable_write(32)
 __gen8_write(8)
 __gen8_write(16)
 __gen8_write(32)
-__hsw_write(8)
-__hsw_write(16)
-__hsw_write(32)
 __gen6_write(8)
 __gen6_write(16)
 __gen6_write(32)
 
-#undef __gen9_write
-#undef __chv_write
+#undef __fwtable_write
 #undef __gen8_write
-#undef __hsw_write
 #undef __gen6_write
 #undef GEN6_WRITE_FOOTER
 #undef GEN6_WRITE_HEADER
@@ -1315,6 +1264,13 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
         WARN_ON(dev_priv->uncore.fw_domains == 0);
 }
 
+#define ASSIGN_FW_DOMAINS_TABLE(d) \
+{ \
+        dev_priv->uncore.fw_domains_table = \
+                        (struct intel_forcewake_range *)(d); \
+        dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
+}
+
 void intel_uncore_init(struct drm_i915_private *dev_priv)
 {
         i915_check_vgpu(dev_priv);
@@ -1328,13 +1284,15 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
         switch (INTEL_INFO(dev_priv)->gen) {
         default:
         case 9:
-                ASSIGN_WRITE_MMIO_VFUNCS(gen9);
-                ASSIGN_READ_MMIO_VFUNCS(gen9);
+                ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
+                ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
+                ASSIGN_READ_MMIO_VFUNCS(fwtable);
                 break;
         case 8:
                 if (IS_CHERRYVIEW(dev_priv)) {
-                        ASSIGN_WRITE_MMIO_VFUNCS(chv);
-                        ASSIGN_READ_MMIO_VFUNCS(chv);
+                        ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
+                        ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
+                        ASSIGN_READ_MMIO_VFUNCS(fwtable);
 
                 } else {
                         ASSIGN_WRITE_MMIO_VFUNCS(gen8);
@@ -1343,14 +1301,11 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
                 break;
         case 7:
         case 6:
-                if (IS_HASWELL(dev_priv)) {
-                        ASSIGN_WRITE_MMIO_VFUNCS(hsw);
-                } else {
-                        ASSIGN_WRITE_MMIO_VFUNCS(gen6);
-                }
+                ASSIGN_WRITE_MMIO_VFUNCS(gen6);
 
                 if (IS_VALLEYVIEW(dev_priv)) {
-                        ASSIGN_READ_MMIO_VFUNCS(vlv);
+                        ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
+                        ASSIGN_READ_MMIO_VFUNCS(fwtable);
                 } else {
                         ASSIGN_READ_MMIO_VFUNCS(gen6);
                 }
@@ -1367,6 +1322,10 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
                 break;
         }
 
+        intel_fw_table_check(dev_priv);
+        if (INTEL_GEN(dev_priv) >= 8)
+                intel_shadow_table_check();
+
         if (intel_vgpu_active(dev_priv)) {
                 ASSIGN_WRITE_MMIO_VFUNCS(vgpu);
                 ASSIGN_READ_MMIO_VFUNCS(vgpu);
@@ -1816,35 +1775,16 @@ static enum forcewake_domains
 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
                                 i915_reg_t reg)
 {
+        u32 offset = i915_mmio_reg_offset(reg);
         enum forcewake_domains fw_domains;
 
-        if (intel_vgpu_active(dev_priv))
-                return 0;
-
-        switch (INTEL_GEN(dev_priv)) {
-        case 9:
-                fw_domains = __gen9_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-                break;
-        case 8:
-                if (IS_CHERRYVIEW(dev_priv))
-                        fw_domains = __chv_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-                else
-                        fw_domains = __gen6_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-                break;
-        case 7:
-        case 6:
-                if (IS_VALLEYVIEW(dev_priv))
-                        fw_domains = __vlv_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-                else
-                        fw_domains = __gen6_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-                break;
-        default:
-                MISSING_CASE(INTEL_INFO(dev_priv)->gen);
-        case 5: /* forcewake was introduced with gen6 */
-        case 4:
-        case 3:
-        case 2:
-                return 0;
+        if (HAS_FWTABLE(dev_priv)) {
+                fw_domains = __fwtable_reg_read_fw_domains(offset);
+        } else if (INTEL_GEN(dev_priv) >= 6) {
+                fw_domains = __gen6_reg_read_fw_domains(offset);
+        } else {
+                WARN_ON(!IS_GEN(dev_priv, 2, 5));
+                fw_domains = 0;
         }
 
         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
@@ -1856,32 +1796,18 @@ static enum forcewake_domains
 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
                                  i915_reg_t reg)
 {
+        u32 offset = i915_mmio_reg_offset(reg);
         enum forcewake_domains fw_domains;
 
-        if (intel_vgpu_active(dev_priv))
-                return 0;
-
-        switch (INTEL_GEN(dev_priv)) {
-        case 9:
-                fw_domains = __gen9_reg_write_fw_domains(i915_mmio_reg_offset(reg));
-                break;
-        case 8:
-                if (IS_CHERRYVIEW(dev_priv))
-                        fw_domains = __chv_reg_write_fw_domains(i915_mmio_reg_offset(reg));
-                else
-                        fw_domains = __gen8_reg_write_fw_domains(i915_mmio_reg_offset(reg));
-                break;
-        case 7:
-        case 6:
+        if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
+                fw_domains = __fwtable_reg_write_fw_domains(offset);
+        } else if (IS_GEN8(dev_priv)) {
+                fw_domains = __gen8_reg_write_fw_domains(offset);
+        } else if (IS_GEN(dev_priv, 6, 7)) {
                 fw_domains = FORCEWAKE_RENDER;
-                break;
-        default:
-                MISSING_CASE(INTEL_INFO(dev_priv)->gen);
-        case 5:
-        case 4:
-        case 3:
-        case 2:
-                return 0;
+        } else {
+                WARN_ON(!IS_GEN(dev_priv, 2, 5));
+                fw_domains = 0;
         }
 
         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
@@ -1911,6 +1837,9 @@ intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
 
         WARN_ON(!op);
 
+        if (intel_vgpu_active(dev_priv))
+                return 0;
+
         if (op & FW_REG_READ)
                 fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
 
diff --git a/drivers/gpu/drm/msm/msm_perf.c b/drivers/gpu/drm/msm/msm_perf.c
index 17fe4e53e0d1..1627294575cb 100644
--- a/drivers/gpu/drm/msm/msm_perf.c
+++ b/drivers/gpu/drm/msm/msm_perf.c
@@ -229,8 +229,8 @@ int msm_perf_debugfs_init(struct drm_minor *minor)
         perf->ent = debugfs_create_file("perf", S_IFREG | S_IRUGO,
                         minor->debugfs_root, perf, &perf_debugfs_fops);
         if (!perf->ent) {
-                DRM_ERROR("Cannot create /sys/kernel/debug/dri/%s/perf\n",
-                                minor->debugfs_root->d_name.name);
+                DRM_ERROR("Cannot create /sys/kernel/debug/dri/%pd/perf\n",
+                                minor->debugfs_root);
                 goto fail;
         }
 
diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c
index 3a5fdfcd67ae..8487f461f05f 100644
--- a/drivers/gpu/drm/msm/msm_rd.c
+++ b/drivers/gpu/drm/msm/msm_rd.c
@@ -243,8 +243,8 @@ int msm_rd_debugfs_init(struct drm_minor *minor)
         rd->ent = debugfs_create_file("rd", S_IFREG | S_IRUGO,
                         minor->debugfs_root, rd, &rd_debugfs_fops);
         if (!rd->ent) {
-                DRM_ERROR("Cannot create /sys/kernel/debug/dri/%s/rd\n",
-                                minor->debugfs_root->d_name.name);
+                DRM_ERROR("Cannot create /sys/kernel/debug/dri/%pd/rd\n",
+                                minor->debugfs_root);
                 goto fail;
         }
 
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
index 7ea8aa7ca408..6bc712f32c8b 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
@@ -175,6 +175,7 @@ struct nvkm_device_func {
         void (*fini)(struct nvkm_device *, bool suspend);
         resource_size_t (*resource_addr)(struct nvkm_device *, unsigned bar);
         resource_size_t (*resource_size)(struct nvkm_device *, unsigned bar);
+        bool cpu_coherent;
 };
 
 struct nvkm_device_quirk {
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 66f31c3eb8ba..343b8659472c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -209,7 +209,8 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
         nvbo->tile_flags = tile_flags;
         nvbo->bo.bdev = &drm->ttm.bdev;
 
-        nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
+        if (!nvxx_device(&drm->device)->func->cpu_coherent)
+                nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
 
         nvbo->page_shift = 12;
         if (drm->client.vm) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
index b1b693219db3..62ad0300cfa5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
@@ -1614,6 +1614,7 @@ nvkm_device_pci_func = {
         .fini = nvkm_device_pci_fini,
         .resource_addr = nvkm_device_pci_resource_addr,
         .resource_size = nvkm_device_pci_resource_size,
+        .cpu_coherent = !IS_ENABLED(CONFIG_ARM),
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
index 939682f18788..9b638bd905ff 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
@@ -245,6 +245,7 @@ nvkm_device_tegra_func = {
         .fini = nvkm_device_tegra_fini,
         .resource_addr = nvkm_device_tegra_resource_addr,
         .resource_size = nvkm_device_tegra_resource_size,
+        .cpu_coherent = false,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
index edec30fd3ecd..0a7b6ed5ed28 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
@@ -37,7 +37,10 @@ nv04_fifo_dma_object_dtor(struct nvkm_fifo_chan *base, int cookie)
 {
         struct nv04_fifo_chan *chan = nv04_fifo_chan(base);
         struct nvkm_instmem *imem = chan->fifo->base.engine.subdev.device->imem;
+
+        mutex_lock(&chan->fifo->base.engine.subdev.mutex);
         nvkm_ramht_remove(imem->ramht, cookie);
+        mutex_unlock(&chan->fifo->base.engine.subdev.mutex);
 }
 
 static int
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index a4e9f35da3a2..74f99bac08b1 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -1638,8 +1638,8 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
         WREG32(AVIVO_D1MODE_VIEWPORT_SIZE + radeon_crtc->crtc_offset,
                (viewport_w << 16) | viewport_h);
 
-        /* set pageflip to happen anywhere in vblank interval */
-        WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
+        /* set pageflip to happen only at start of vblank interval (front porch) */
+        WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 3);
 
         if (!atomic && fb && fb != crtc->primary->fb) {
                 radeon_fb = to_radeon_framebuffer(fb);
diff --git a/drivers/gpu/drm/radeon/r600_dpm.c b/drivers/gpu/drm/radeon/r600_dpm.c
index 6a4b020dd0b4..5a26eb4545aa 100644
--- a/drivers/gpu/drm/radeon/r600_dpm.c
+++ b/drivers/gpu/drm/radeon/r600_dpm.c
@@ -156,19 +156,20 @@ u32 r600_dpm_get_vblank_time(struct radeon_device *rdev)
         struct drm_device *dev = rdev->ddev;
         struct drm_crtc *crtc;
         struct radeon_crtc *radeon_crtc;
-        u32 line_time_us, vblank_lines;
+        u32 vblank_in_pixels;
         u32 vblank_time_us = 0xffffffff; /* if the displays are off, vblank time is max */
 
         if (rdev->num_crtc && rdev->mode_info.mode_config_initialized) {
                 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                         radeon_crtc = to_radeon_crtc(crtc);
                         if (crtc->enabled && radeon_crtc->enabled && radeon_crtc->hw_mode.clock) {
-                                line_time_us = (radeon_crtc->hw_mode.crtc_htotal * 1000) /
-                                        radeon_crtc->hw_mode.clock;
-                                vblank_lines = radeon_crtc->hw_mode.crtc_vblank_end -
-                                        radeon_crtc->hw_mode.crtc_vdisplay +
-                                        (radeon_crtc->v_border * 2);
-                                vblank_time_us = vblank_lines * line_time_us;
+                                vblank_in_pixels =
+                                        radeon_crtc->hw_mode.crtc_htotal *
+                                        (radeon_crtc->hw_mode.crtc_vblank_end -
+                                         radeon_crtc->hw_mode.crtc_vdisplay +
+                                         (radeon_crtc->v_border * 2));
+
+                                vblank_time_us = vblank_in_pixels * 1000 / radeon_crtc->hw_mode.clock;
                                 break;
                         }
                 }
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 50e96d2c593d..e18839d52e3e 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -927,6 +927,16 @@ radeon_lvds_detect(struct drm_connector *connector, bool force)
         return ret;
 }
 
+static void radeon_connector_unregister(struct drm_connector *connector)
+{
+        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+
+        if (radeon_connector->ddc_bus->has_aux) {
+                drm_dp_aux_unregister(&radeon_connector->ddc_bus->aux);
+                radeon_connector->ddc_bus->has_aux = false;
+        }
+}
+
 static void radeon_connector_destroy(struct drm_connector *connector)
 {
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -984,6 +994,7 @@ static const struct drm_connector_funcs radeon_lvds_connector_funcs = {
         .dpms = drm_helper_connector_dpms,
         .detect = radeon_lvds_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .early_unregister = radeon_connector_unregister,
         .destroy = radeon_connector_destroy,
         .set_property = radeon_lvds_set_property,
 };
@@ -1111,6 +1122,7 @@ static const struct drm_connector_funcs radeon_vga_connector_funcs = {
         .dpms = drm_helper_connector_dpms,
         .detect = radeon_vga_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .early_unregister = radeon_connector_unregister,
         .destroy = radeon_connector_destroy,
         .set_property = radeon_connector_set_property,
 };
@@ -1188,6 +1200,7 @@ static const struct drm_connector_funcs radeon_tv_connector_funcs = {
         .dpms = drm_helper_connector_dpms,
         .detect = radeon_tv_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .early_unregister = radeon_connector_unregister,
         .destroy = radeon_connector_destroy,
         .set_property = radeon_connector_set_property,
 };
@@ -1519,6 +1532,7 @@ static const struct drm_connector_funcs radeon_dvi_connector_funcs = {
         .detect = radeon_dvi_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = radeon_connector_set_property,
+        .early_unregister = radeon_connector_unregister,
         .destroy = radeon_connector_destroy,
         .force = radeon_dvi_force,
 };
@@ -1832,6 +1846,7 @@ static const struct drm_connector_funcs radeon_dp_connector_funcs = {
         .detect = radeon_dp_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = radeon_connector_set_property,
+        .early_unregister = radeon_connector_unregister,
         .destroy = radeon_connector_destroy,
         .force = radeon_dvi_force,
 };
@@ -1841,6 +1856,7 @@ static const struct drm_connector_funcs radeon_edp_connector_funcs = {
         .detect = radeon_dp_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = radeon_lvds_set_property,
+        .early_unregister = radeon_connector_unregister,
         .destroy = radeon_connector_destroy,
         .force = radeon_dvi_force,
 };
@@ -1850,6 +1866,7 @@ static const struct drm_connector_funcs radeon_lvds_bridge_connector_funcs = {
         .detect = radeon_dp_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = radeon_lvds_set_property,
+        .early_unregister = radeon_connector_unregister,
         .destroy = radeon_connector_destroy,
         .force = radeon_dvi_force,
 };
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 890171f08987..cdb8cb568c15 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -321,16 +321,30 @@ void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id)
         update_pending = radeon_page_flip_pending(rdev, crtc_id);
 
         /* Has the pageflip already completed in crtc, or is it certain
-         * to complete in this vblank?
+         * to complete in this vblank? GET_DISTANCE_TO_VBLANKSTART provides
+         * distance to start of "fudged earlier" vblank in vpos, distance to
+         * start of real vblank in hpos. vpos >= 0 && hpos < 0 means we are in
+         * the last few scanlines before start of real vblank, where the vblank
+         * irq can fire, so we have sampled update_pending a bit too early and
+         * know the flip will complete at leading edge of the upcoming real
+         * vblank. On pre-AVIVO hardware, flips also complete inside the real
+         * vblank, not only at leading edge, so if update_pending for hpos >= 0
+         *  == inside real vblank, the flip will complete almost immediately.
+         * Note that this method of completion handling is still not 100% race
+         * free, as we could execute before the radeon_flip_work_func managed
+         * to run and set the RADEON_FLIP_SUBMITTED status, thereby we no-op,
+         * but the flip still gets programmed into hw and completed during
+         * vblank, leading to a delayed emission of the flip completion event.
+         * This applies at least to pre-AVIVO hardware, where flips are always
+         * completing inside vblank, not only at leading edge of vblank.
          */
         if (update_pending &&
-            (DRM_SCANOUTPOS_VALID & radeon_get_crtc_scanoutpos(rdev->ddev,
-                                                               crtc_id,
-                                                               USE_REAL_VBLANKSTART,
-                                                               &vpos, &hpos, NULL, NULL,
-                                                               &rdev->mode_info.crtcs[crtc_id]->base.hwmode)) &&
-            ((vpos >= (99 * rdev->mode_info.crtcs[crtc_id]->base.hwmode.crtc_vdisplay)/100) ||
-             (vpos < 0 && !ASIC_IS_AVIVO(rdev)))) {
+            (DRM_SCANOUTPOS_VALID &
+             radeon_get_crtc_scanoutpos(rdev->ddev, crtc_id,
+                                        GET_DISTANCE_TO_VBLANKSTART,
+                                        &vpos, &hpos, NULL, NULL,
+                                        &rdev->mode_info.crtcs[crtc_id]->base.hwmode)) &&
+            ((vpos >= 0 && hpos < 0) || (hpos >= 0 && !ASIC_IS_AVIVO(rdev)))) {
                 /* crtc didn't flip in this target vblank interval,
                  * but flip is pending in crtc. Based on the current
                  * scanout position we know that the current frame is
@@ -438,16 +452,19 @@ static void radeon_flip_work_func(struct work_struct *__work)
         }
 
         /* Wait until we're out of the vertical blank period before the one
-         * targeted by the flip
+         * targeted by the flip. Always wait on pre DCE4 to avoid races with
+         * flip completion handling from vblank irq, as these old asics don't
+         * have reliable pageflip completion interrupts.
          */
         while (radeon_crtc->enabled &&
-               (radeon_get_crtc_scanoutpos(dev, work->crtc_id, 0,
-                                           &vpos, &hpos, NULL, NULL,
-                                           &crtc->hwmode)
+                (radeon_get_crtc_scanoutpos(dev, work->crtc_id, 0,
+                                            &vpos, &hpos, NULL, NULL,
+                                            &crtc->hwmode)
                 & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) ==
-               (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) &&
-               (int)(work->target_vblank -
-                     dev->driver->get_vblank_counter(dev, work->crtc_id)) > 0)
+                (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) &&
+                (!ASIC_IS_AVIVO(rdev) ||
+                ((int) (work->target_vblank -
+                dev->driver->get_vblank_counter(dev, work->crtc_id)) > 0)))
                 usleep_range(1000, 2000);
 
         /* We borrow the event spin lock for protecting flip_status */
@@ -1658,20 +1675,20 @@ int radeon_modeset_init(struct radeon_device *rdev)
 
 void radeon_modeset_fini(struct radeon_device *rdev)
 {
-        radeon_fbdev_fini(rdev);
-        kfree(rdev->mode_info.bios_hardcoded_edid);
-
-        /* free i2c buses */
-        radeon_i2c_fini(rdev);
-
         if (rdev->mode_info.mode_config_initialized) {
-                radeon_afmt_fini(rdev);
                 drm_kms_helper_poll_fini(rdev->ddev);
                 radeon_hpd_fini(rdev);
                 drm_crtc_force_disable_all(rdev->ddev);
+                radeon_fbdev_fini(rdev);
+                radeon_afmt_fini(rdev);
                 drm_mode_config_cleanup(rdev->ddev);
                 rdev->mode_info.mode_config_initialized = false;
         }
+
+        kfree(rdev->mode_info.bios_hardcoded_edid);
+
+        /* free i2c buses */
+        radeon_i2c_fini(rdev);
 }
 
 static bool is_hdtv_mode(const struct drm_display_mode *mode)
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 91c8f4339566..00ea0002b539 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -96,9 +96,10 @@
  *   2.45.0 - Allow setting shader registers using DMA/COPY packet3 on SI
  *   2.46.0 - Add PFP_SYNC_ME support on evergreen
  *   2.47.0 - Add UVD_NO_OP register support
+ *   2.48.0 - TA_CS_BC_BASE_ADDR allowed on SI
  */
 #define KMS_DRIVER_MAJOR        2
-#define KMS_DRIVER_MINOR        47
+#define KMS_DRIVER_MINOR        48
 #define KMS_DRIVER_PATCHLEVEL   0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
diff --git a/drivers/gpu/drm/radeon/radeon_i2c.c b/drivers/gpu/drm/radeon/radeon_i2c.c
index 021aa005623f..29f7817af821 100644
--- a/drivers/gpu/drm/radeon/radeon_i2c.c
+++ b/drivers/gpu/drm/radeon/radeon_i2c.c
@@ -982,9 +982,8 @@ void radeon_i2c_destroy(struct radeon_i2c_chan *i2c)
 {
         if (!i2c)
                 return;
+        WARN_ON(i2c->has_aux);
         i2c_del_adapter(&i2c->adapter);
-        if (i2c->has_aux)
-                drm_dp_aux_unregister(&i2c->aux);
         kfree(i2c);
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 455268214b89..3de5e6e21662 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -566,7 +566,8 @@ static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm)
                 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
                 struct page **pages = ttm->pages + pinned;
 
-                r = get_user_pages(userptr, num_pages, write, 0, pages, NULL);
+                r = get_user_pages(userptr, num_pages, write ? FOLL_WRITE : 0,
+                                   pages, NULL);
                 if (r < 0)
                         goto release_pages;
 
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 76c55c5d11ec..c55d653aaf5f 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -406,8 +406,9 @@ void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save)
         for (i = 0; i < rdev->num_crtc; i++) {
                 if (save->crtc_enabled[i]) {
                         tmp = RREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + crtc_offsets[i]);
-                        if ((tmp & 0x7) != 0) {
+                        if ((tmp & 0x7) != 3) {
                                 tmp &= ~0x7;
+                                tmp |= 0x3;
                                 WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
                         }
                         tmp = RREG32(AVIVO_D1GRPH_UPDATE + crtc_offsets[i]);
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index 7ee9aafbdf74..e402be8821c4 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -4431,6 +4431,7 @@ static bool si_vm_reg_valid(u32 reg)
         case SPI_CONFIG_CNTL:
         case SPI_CONFIG_CNTL_1:
         case TA_CNTL_AUX:
+        case TA_CS_BC_BASE_ADDR:
                 return true;
         default:
                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index ba7e3d19b38d..89bdf20344ae 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -3015,6 +3015,12 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
         if (rdev->pdev->device == 0x6811 &&
             rdev->pdev->revision == 0x81)
                 max_mclk = 120000;
+        /* limit sclk/mclk on Jet parts for stability */
+        if (rdev->pdev->device == 0x6665 &&
+            rdev->pdev->revision == 0xc3) {
+                max_sclk = 75000;
+                max_mclk = 80000;
+        }
 
         if (rps->vce_active) {
                 rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index eb220eecba78..65a911ddd509 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -1145,6 +1145,7 @@
 #define SPI_LB_CU_MASK                                  0x9354
 
 #define TA_CNTL_AUX                                     0x9508
+#define TA_CS_BC_BASE_ADDR                              0x950C
 
 #define CC_RB_BACKEND_DISABLE                           0x98F4
 #define         BACKEND_DISABLE(x)                      ((x) << 16)
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index 9688bfa92ccd..611b6b9bb3cb 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -122,7 +122,7 @@ int udl_handle_damage(struct udl_framebuffer *fb, int x, int y,
                 return 0;
         cmd = urb->transfer_buffer;
 
-        for (i = y; i < height ; i++) {
+        for (i = y; i < y + height ; i++) {
                 const int line_offset = fb->base.pitches[0] * i;
                 const int byte_offset = line_offset + (x * bpp);
                 const int dev_byte_offset = (fb->base.width * bpp * i) + (x * bpp);
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 2682f07d8f1e..7f08d681a74b 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -229,7 +229,7 @@ int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
          * and need to make things up in a approximative but consistent way.
          */
         ret |= DRM_SCANOUTPOS_IN_VBLANK;
-        vblank_lines = mode->crtc_vtotal - mode->crtc_vdisplay;
+        vblank_lines = mode->vtotal - mode->vdisplay;
 
         if (flags & DRM_CALLED_FROM_VBLIRQ) {
                 /*
@@ -378,7 +378,7 @@ static void vc4_crtc_mode_set_nofb(struct drm_crtc *crtc)
         struct drm_crtc_state *state = crtc->state;
         struct drm_display_mode *mode = &state->adjusted_mode;
         bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE;
-        u32 vactive = (mode->vdisplay >> (interlace ? 1 : 0));
+        u32 pixel_rep = (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1;
         u32 format = PV_CONTROL_FORMAT_24;
         bool debug_dump_regs = false;
         int clock_select = vc4_get_clock_select(crtc);
@@ -394,47 +394,65 @@ static void vc4_crtc_mode_set_nofb(struct drm_crtc *crtc)
         CRTC_WRITE(PV_CONTROL, 0);
 
         CRTC_WRITE(PV_HORZA,
-                   VC4_SET_FIELD(mode->htotal - mode->hsync_end,
+                   VC4_SET_FIELD((mode->htotal -
+                                  mode->hsync_end) * pixel_rep,
                                  PV_HORZA_HBP) |
-                   VC4_SET_FIELD(mode->hsync_end - mode->hsync_start,
+                   VC4_SET_FIELD((mode->hsync_end -
+                                  mode->hsync_start) * pixel_rep,
                                  PV_HORZA_HSYNC));
         CRTC_WRITE(PV_HORZB,
-                   VC4_SET_FIELD(mode->hsync_start - mode->hdisplay,
+                   VC4_SET_FIELD((mode->hsync_start -
+                                  mode->hdisplay) * pixel_rep,
                                  PV_HORZB_HFP) |
-                   VC4_SET_FIELD(mode->hdisplay, PV_HORZB_HACTIVE));
+                   VC4_SET_FIELD(mode->hdisplay * pixel_rep, PV_HORZB_HACTIVE));
 
         CRTC_WRITE(PV_VERTA,
-                   VC4_SET_FIELD(mode->vtotal - mode->vsync_end,
+                   VC4_SET_FIELD(mode->crtc_vtotal - mode->crtc_vsync_end,
                                  PV_VERTA_VBP) |
-                   VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+                   VC4_SET_FIELD(mode->crtc_vsync_end - mode->crtc_vsync_start,
                                  PV_VERTA_VSYNC));
         CRTC_WRITE(PV_VERTB,
-                   VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+                   VC4_SET_FIELD(mode->crtc_vsync_start - mode->crtc_vdisplay,
                                  PV_VERTB_VFP) |
-                   VC4_SET_FIELD(vactive, PV_VERTB_VACTIVE));
+                   VC4_SET_FIELD(mode->crtc_vdisplay, PV_VERTB_VACTIVE));
 
         if (interlace) {
                 CRTC_WRITE(PV_VERTA_EVEN,
-                           VC4_SET_FIELD(mode->vtotal - mode->vsync_end - 1,
+                           VC4_SET_FIELD(mode->crtc_vtotal -
+                                         mode->crtc_vsync_end - 1,
                                          PV_VERTA_VBP) |
-                           VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+                           VC4_SET_FIELD(mode->crtc_vsync_end -
+                                         mode->crtc_vsync_start,
                                          PV_VERTA_VSYNC));
                 CRTC_WRITE(PV_VERTB_EVEN,
-                           VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+                           VC4_SET_FIELD(mode->crtc_vsync_start -
+                                         mode->crtc_vdisplay,
                                          PV_VERTB_VFP) |
-                           VC4_SET_FIELD(vactive, PV_VERTB_VACTIVE));
+                           VC4_SET_FIELD(mode->crtc_vdisplay, PV_VERTB_VACTIVE));
+
+                /* We set up first field even mode for HDMI.  VEC's
+                 * NTSC mode would want first field odd instead, once
+                 * we support it (to do so, set ODD_FIRST and put the
+                 * delay in VSYNCD_EVEN instead).
+                 */
+                CRTC_WRITE(PV_V_CONTROL,
+                           PV_VCONTROL_CONTINUOUS |
+                           PV_VCONTROL_INTERLACE |
+                           VC4_SET_FIELD(mode->htotal * pixel_rep / 2,
+                                         PV_VCONTROL_ODD_DELAY));
+                CRTC_WRITE(PV_VSYNCD_EVEN, 0);
+        } else {
+                CRTC_WRITE(PV_V_CONTROL, PV_VCONTROL_CONTINUOUS);
         }
 
-        CRTC_WRITE(PV_HACT_ACT, mode->hdisplay);
+        CRTC_WRITE(PV_HACT_ACT, mode->hdisplay * pixel_rep);
 
-        CRTC_WRITE(PV_V_CONTROL,
-                   PV_VCONTROL_CONTINUOUS |
-                   (interlace ? PV_VCONTROL_INTERLACE : 0));
 
         CRTC_WRITE(PV_CONTROL,
                    VC4_SET_FIELD(format, PV_CONTROL_FORMAT) |
                    VC4_SET_FIELD(vc4_get_fifo_full_level(format),
                                  PV_CONTROL_FIFO_LEVEL) |
+                   VC4_SET_FIELD(pixel_rep - 1, PV_CONTROL_PIXEL_REP) |
                    PV_CONTROL_CLR_AT_START |
                    PV_CONTROL_TRIGGER_UNDERFLOW |
                    PV_CONTROL_WAIT_HSTART |
@@ -544,16 +562,6 @@ static bool vc4_crtc_mode_fixup(struct drm_crtc *crtc,
                 return false;
         }
 
-        /*
-         * Interlaced video modes got CRTC_INTERLACE_HALVE_V applied when
-         * coming from user space. We don't want this, as it screws up
-         * vblank timestamping, so fix it up.
-         */
-        drm_mode_set_crtcinfo(adjusted_mode, 0);
-
-        DRM_DEBUG_KMS("[CRTC:%d] adjusted_mode :\n", crtc->base.id);
-        drm_mode_debug_printmodeline(adjusted_mode);
-
         return true;
 }
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 428e24919ef1..7c1e4d97486f 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -122,9 +122,16 @@ to_vc4_dev(struct drm_device *dev)
 struct vc4_bo {
         struct drm_gem_cma_object base;
 
-        /* seqno of the last job to render to this BO. */
+        /* seqno of the last job to render using this BO. */
         uint64_t seqno;
 
+        /* seqno of the last job to use the RCL to write to this BO.
+         *
+         * Note that this doesn't include binner overflow memory
+         * writes.
+         */
+        uint64_t write_seqno;
+
         /* List entry for the BO's position in either
          * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list
          */
@@ -216,6 +223,9 @@ struct vc4_exec_info {
         /* Sequence number for this bin/render job. */
         uint64_t seqno;
 
+        /* Latest write_seqno of any BO that binning depends on. */
+        uint64_t bin_dep_seqno;
+
         /* Last current addresses the hardware was processing when the
          * hangcheck timer checked on us.
          */
@@ -230,6 +240,13 @@ struct vc4_exec_info {
         struct drm_gem_cma_object **bo;
         uint32_t bo_count;
 
+        /* List of BOs that are being written by the RCL.  Other than
+         * the binner temporary storage, this is all the BOs written
+         * by the job.
+         */
+        struct drm_gem_cma_object *rcl_write_bo[4];
+        uint32_t rcl_write_bo_count;
+
         /* Pointers for our position in vc4->job_list */
         struct list_head head;
 
@@ -307,18 +324,15 @@ struct vc4_exec_info {
 static inline struct vc4_exec_info *
 vc4_first_bin_job(struct vc4_dev *vc4)
 {
-        if (list_empty(&vc4->bin_job_list))
-                return NULL;
-        return list_first_entry(&vc4->bin_job_list, struct vc4_exec_info, head);
+        return list_first_entry_or_null(&vc4->bin_job_list,
+                                        struct vc4_exec_info, head);
 }
 
 static inline struct vc4_exec_info *
 vc4_first_render_job(struct vc4_dev *vc4)
 {
-        if (list_empty(&vc4->render_job_list))
-                return NULL;
-        return list_first_entry(&vc4->render_job_list,
-                                struct vc4_exec_info, head);
+        return list_first_entry_or_null(&vc4->render_job_list,
+                                        struct vc4_exec_info, head);
 }
 
 static inline struct vc4_exec_info *
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index 77daea6cb866..47a095f392f8 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -467,6 +467,11 @@ vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
         list_for_each_entry(bo, &exec->unref_list, unref_head) {
                 bo->seqno = seqno;
         }
+
+        for (i = 0; i < exec->rcl_write_bo_count; i++) {
+                bo = to_vc4_bo(&exec->rcl_write_bo[i]->base);
+                bo->write_seqno = seqno;
+        }
 }
 
 /* Queues a struct vc4_exec_info for execution.  If no job is
@@ -669,6 +674,14 @@ vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
                 goto fail;
 
         ret = vc4_validate_shader_recs(dev, exec);
+        if (ret)
+                goto fail;
+
+        /* Block waiting on any previous rendering into the CS's VBO,
+         * IB, or textures, so that pixels are actually written by the
+         * time we try to read them.
+         */
+        ret = vc4_wait_for_seqno(dev, exec->bin_dep_seqno, ~0ull, true);
 
 fail:
         drm_free_large(temp);
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index 68ad10634b29..c4cb2e26de32 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -62,6 +62,8 @@ struct vc4_hdmi {
 struct vc4_hdmi_encoder {
         struct vc4_encoder base;
         bool hdmi_monitor;
+        bool limited_rgb_range;
+        bool rgb_range_selectable;
 };
 
 static inline struct vc4_hdmi_encoder *
@@ -174,6 +176,9 @@ vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
                         return connector_status_disconnected;
         }
 
+        if (drm_probe_ddc(vc4->hdmi->ddc))
+                return connector_status_connected;
+
         if (HDMI_READ(VC4_HDMI_HOTPLUG) & VC4_HDMI_HOTPLUG_CONNECTED)
                 return connector_status_connected;
         else
@@ -202,41 +207,22 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
                 return -ENODEV;
 
         vc4_encoder->hdmi_monitor = drm_detect_hdmi_monitor(edid);
+
+        if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) {
+                vc4_encoder->rgb_range_selectable =
+                        drm_rgb_quant_range_selectable(edid);
+        }
+
         drm_mode_connector_update_edid_property(connector, edid);
         ret = drm_add_edid_modes(connector, edid);
 
         return ret;
 }
 
-/*
- * drm_helper_probe_single_connector_modes() applies drm_mode_set_crtcinfo to
- * all modes with flag CRTC_INTERLACE_HALVE_V. We don't want this, as it
- * screws up vblank timestamping for interlaced modes, so fix it up.
- */
-static int vc4_hdmi_connector_probe_modes(struct drm_connector *connector,
-                                          uint32_t maxX, uint32_t maxY)
-{
-        struct drm_display_mode *mode;
-        int count;
-
-        count = drm_helper_probe_single_connector_modes(connector, maxX, maxY);
-        if (count == 0)
-                return 0;
-
-        DRM_DEBUG_KMS("[CONNECTOR:%d:%s] probed adapted modes :\n",
-                      connector->base.id, connector->name);
-        list_for_each_entry(mode, &connector->modes, head) {
-                drm_mode_set_crtcinfo(mode, 0);
-                drm_mode_debug_printmodeline(mode);
-        }
-
-        return count;
-}
-
 static const struct drm_connector_funcs vc4_hdmi_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = vc4_hdmi_connector_detect,
-        .fill_modes = vc4_hdmi_connector_probe_modes,
+        .fill_modes = drm_helper_probe_single_connector_modes,
         .destroy = vc4_hdmi_connector_destroy,
         .reset = drm_atomic_helper_connector_reset,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -294,25 +280,143 @@ static const struct drm_encoder_funcs vc4_hdmi_encoder_funcs = {
         .destroy = vc4_hdmi_encoder_destroy,
 };
 
+static int vc4_hdmi_stop_packet(struct drm_encoder *encoder,
+                                enum hdmi_infoframe_type type)
+{
+        struct drm_device *dev = encoder->dev;
+        struct vc4_dev *vc4 = to_vc4_dev(dev);
+        u32 packet_id = type - 0x80;
+
+        HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG,
+                   HDMI_READ(VC4_HDMI_RAM_PACKET_CONFIG) & ~BIT(packet_id));
+
+        return wait_for(!(HDMI_READ(VC4_HDMI_RAM_PACKET_STATUS) &
+                          BIT(packet_id)), 100);
+}
+
+static void vc4_hdmi_write_infoframe(struct drm_encoder *encoder,
+                                     union hdmi_infoframe *frame)
+{
+        struct drm_device *dev = encoder->dev;
+        struct vc4_dev *vc4 = to_vc4_dev(dev);
+        u32 packet_id = frame->any.type - 0x80;
+        u32 packet_reg = VC4_HDMI_GCP_0 + VC4_HDMI_PACKET_STRIDE * packet_id;
+        uint8_t buffer[VC4_HDMI_PACKET_STRIDE];
+        ssize_t len, i;
+        int ret;
+
+        WARN_ONCE(!(HDMI_READ(VC4_HDMI_RAM_PACKET_CONFIG) &
+                    VC4_HDMI_RAM_PACKET_ENABLE),
+                  "Packet RAM has to be on to store the packet.");
+
+        len = hdmi_infoframe_pack(frame, buffer, sizeof(buffer));
+        if (len < 0)
+                return;
+
+        ret = vc4_hdmi_stop_packet(encoder, frame->any.type);
+        if (ret) {
+                DRM_ERROR("Failed to wait for infoframe to go idle: %d\n", ret);
+                return;
+        }
+
+        for (i = 0; i < len; i += 7) {
+                HDMI_WRITE(packet_reg,
+                           buffer[i + 0] << 0 |
+                           buffer[i + 1] << 8 |
+                           buffer[i + 2] << 16);
+                packet_reg += 4;
+
+                HDMI_WRITE(packet_reg,
+                           buffer[i + 3] << 0 |
+                           buffer[i + 4] << 8 |
+                           buffer[i + 5] << 16 |
+                           buffer[i + 6] << 24);
+                packet_reg += 4;
+        }
+
+        HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG,
+                   HDMI_READ(VC4_HDMI_RAM_PACKET_CONFIG) | BIT(packet_id));
+        ret = wait_for((HDMI_READ(VC4_HDMI_RAM_PACKET_STATUS) &
+                        BIT(packet_id)), 100);
+        if (ret)
+                DRM_ERROR("Failed to wait for infoframe to start: %d\n", ret);
+}
+
+static void vc4_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
+{
+        struct vc4_hdmi_encoder *vc4_encoder = to_vc4_hdmi_encoder(encoder);
+        struct drm_crtc *crtc = encoder->crtc;
+        const struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+        union hdmi_infoframe frame;
+        int ret;
+
+        ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+        if (ret < 0) {
+                DRM_ERROR("couldn't fill AVI infoframe\n");
+                return;
+        }
+
+        if (vc4_encoder->rgb_range_selectable) {
+                if (vc4_encoder->limited_rgb_range) {
+                        frame.avi.quantization_range =
+                                HDMI_QUANTIZATION_RANGE_LIMITED;
+                } else {
+                        frame.avi.quantization_range =
+                                HDMI_QUANTIZATION_RANGE_FULL;
+                }
+        }
+
+        vc4_hdmi_write_infoframe(encoder, &frame);
+}
+
+static void vc4_hdmi_set_spd_infoframe(struct drm_encoder *encoder)
+{
+        union hdmi_infoframe frame;
+        int ret;
+
+        ret = hdmi_spd_infoframe_init(&frame.spd, "Broadcom", "Videocore");
+        if (ret < 0) {
+                DRM_ERROR("couldn't fill SPD infoframe\n");
+                return;
+        }
+
+        frame.spd.sdi = HDMI_SPD_SDI_PC;
+
+        vc4_hdmi_write_infoframe(encoder, &frame);
+}
+
+static void vc4_hdmi_set_infoframes(struct drm_encoder *encoder)
+{
+        vc4_hdmi_set_avi_infoframe(encoder);
+        vc4_hdmi_set_spd_infoframe(encoder);
+}
+
 static void vc4_hdmi_encoder_mode_set(struct drm_encoder *encoder,
                                       struct drm_display_mode *unadjusted_mode,
                                       struct drm_display_mode *mode)
 {
+        struct vc4_hdmi_encoder *vc4_encoder = to_vc4_hdmi_encoder(encoder);
         struct drm_device *dev = encoder->dev;
         struct vc4_dev *vc4 = to_vc4_dev(dev);
         bool debug_dump_regs = false;
         bool hsync_pos = mode->flags & DRM_MODE_FLAG_PHSYNC;
         bool vsync_pos = mode->flags & DRM_MODE_FLAG_PVSYNC;
-        u32 vactive = (mode->vdisplay >>
-                       ((mode->flags & DRM_MODE_FLAG_INTERLACE) ? 1 : 0));
-        u32 verta = (VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+        bool interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE;
+        u32 pixel_rep = (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1;
+        u32 verta = (VC4_SET_FIELD(mode->crtc_vsync_end - mode->crtc_vsync_start,
                                    VC4_HDMI_VERTA_VSP) |
-                     VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+                     VC4_SET_FIELD(mode->crtc_vsync_start - mode->crtc_vdisplay,
                                    VC4_HDMI_VERTA_VFP) |
-                     VC4_SET_FIELD(vactive, VC4_HDMI_VERTA_VAL));
+                     VC4_SET_FIELD(mode->crtc_vdisplay, VC4_HDMI_VERTA_VAL));
         u32 vertb = (VC4_SET_FIELD(0, VC4_HDMI_VERTB_VSPO) |
-                     VC4_SET_FIELD(mode->vtotal - mode->vsync_end,
+                     VC4_SET_FIELD(mode->crtc_vtotal - mode->crtc_vsync_end,
                                    VC4_HDMI_VERTB_VBP));
+        u32 vertb_even = (VC4_SET_FIELD(0, VC4_HDMI_VERTB_VSPO) |
+                          VC4_SET_FIELD(mode->crtc_vtotal -
+                                        mode->crtc_vsync_end -
+                                        interlaced,
+                                        VC4_HDMI_VERTB_VBP));
+        u32 csc_ctl;
 
         if (debug_dump_regs) {
                 DRM_INFO("HDMI regs before:\n");
@@ -321,7 +425,8 @@ static void vc4_hdmi_encoder_mode_set(struct drm_encoder *encoder,
 
         HD_WRITE(VC4_HD_VID_CTL, 0);
 
-        clk_set_rate(vc4->hdmi->pixel_clock, mode->clock * 1000);
+        clk_set_rate(vc4->hdmi->pixel_clock, mode->clock * 1000 *
+                     ((mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1));
 
         HDMI_WRITE(VC4_HDMI_SCHEDULER_CONTROL,
                    HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) |
@@ -331,29 +436,62 @@ static void vc4_hdmi_encoder_mode_set(struct drm_encoder *encoder,
         HDMI_WRITE(VC4_HDMI_HORZA,
                    (vsync_pos ? VC4_HDMI_HORZA_VPOS : 0) |
                    (hsync_pos ? VC4_HDMI_HORZA_HPOS : 0) |
-                   VC4_SET_FIELD(mode->hdisplay, VC4_HDMI_HORZA_HAP));
+                   VC4_SET_FIELD(mode->hdisplay * pixel_rep,
+                                 VC4_HDMI_HORZA_HAP));
 
         HDMI_WRITE(VC4_HDMI_HORZB,
-                   VC4_SET_FIELD(mode->htotal - mode->hsync_end,
+                   VC4_SET_FIELD((mode->htotal -
+                                  mode->hsync_end) * pixel_rep,
                                  VC4_HDMI_HORZB_HBP) |
-                   VC4_SET_FIELD(mode->hsync_end - mode->hsync_start,
+                   VC4_SET_FIELD((mode->hsync_end -
+                                  mode->hsync_start) * pixel_rep,
                                  VC4_HDMI_HORZB_HSP) |
-                   VC4_SET_FIELD(mode->hsync_start - mode->hdisplay,
+                   VC4_SET_FIELD((mode->hsync_start -
+                                  mode->hdisplay) * pixel_rep,
                                  VC4_HDMI_HORZB_HFP));
 
         HDMI_WRITE(VC4_HDMI_VERTA0, verta);
         HDMI_WRITE(VC4_HDMI_VERTA1, verta);
 
-        HDMI_WRITE(VC4_HDMI_VERTB0, vertb);
+        HDMI_WRITE(VC4_HDMI_VERTB0, vertb_even);
         HDMI_WRITE(VC4_HDMI_VERTB1, vertb);
 
         HD_WRITE(VC4_HD_VID_CTL,
                  (vsync_pos ? 0 : VC4_HD_VID_CTL_VSYNC_LOW) |
                  (hsync_pos ? 0 : VC4_HD_VID_CTL_HSYNC_LOW));
 
+        csc_ctl = VC4_SET_FIELD(VC4_HD_CSC_CTL_ORDER_BGR,
+                                VC4_HD_CSC_CTL_ORDER);
+
+        if (vc4_encoder->hdmi_monitor && drm_match_cea_mode(mode) > 1) {
+                /* CEA VICs other than #1 requre limited range RGB
+                 * output unless overridden by an AVI infoframe.
+                 * Apply a colorspace conversion to squash 0-255 down
+                 * to 16-235.  The matrix here is:
+                 *
+                 * [ 0      0      0.8594 16]
+                 * [ 0      0.8594 0      16]
+                 * [ 0.8594 0      0      16]
+                 * [ 0      0      0       1]
+                 */
+                csc_ctl |= VC4_HD_CSC_CTL_ENABLE;
+                csc_ctl |= VC4_HD_CSC_CTL_RGB2YCC;
+                csc_ctl |= VC4_SET_FIELD(VC4_HD_CSC_CTL_MODE_CUSTOM,
+                                         VC4_HD_CSC_CTL_MODE);
+
+                HD_WRITE(VC4_HD_CSC_12_11, (0x000 << 16) | 0x000);
+                HD_WRITE(VC4_HD_CSC_14_13, (0x100 << 16) | 0x6e0);
+                HD_WRITE(VC4_HD_CSC_22_21, (0x6e0 << 16) | 0x000);
+                HD_WRITE(VC4_HD_CSC_24_23, (0x100 << 16) | 0x000);
+                HD_WRITE(VC4_HD_CSC_32_31, (0x000 << 16) | 0x6e0);
+                HD_WRITE(VC4_HD_CSC_34_33, (0x100 << 16) | 0x000);
+                vc4_encoder->limited_rgb_range = true;
+        } else {
+                vc4_encoder->limited_rgb_range = false;
+        }
+
         /* The RGB order applies even when CSC is disabled. */
-        HD_WRITE(VC4_HD_CSC_CTL, VC4_SET_FIELD(VC4_HD_CSC_CTL_ORDER_BGR,
-                                               VC4_HD_CSC_CTL_ORDER));
+        HD_WRITE(VC4_HD_CSC_CTL, csc_ctl);
 
         HDMI_WRITE(VC4_HDMI_FIFO_CTL, VC4_HDMI_FIFO_CTL_MASTER_SLAVE_N);
 
@@ -368,6 +506,8 @@ static void vc4_hdmi_encoder_disable(struct drm_encoder *encoder)
         struct drm_device *dev = encoder->dev;
         struct vc4_dev *vc4 = to_vc4_dev(dev);
 
+        HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG, 0);
+
         HDMI_WRITE(VC4_HDMI_TX_PHY_RESET_CTL, 0xf << 16);
         HD_WRITE(VC4_HD_VID_CTL,
                  HD_READ(VC4_HD_VID_CTL) & ~VC4_HD_VID_CTL_ENABLE);
@@ -394,7 +534,7 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
                            VC4_HDMI_SCHEDULER_CONTROL_MODE_HDMI);
 
                 ret = wait_for(HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
-                               VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE, 1);
+                               VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE, 1000);
                 WARN_ONCE(ret, "Timeout waiting for "
                           "VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE\n");
         } else {
@@ -406,7 +546,7 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
                            ~VC4_HDMI_SCHEDULER_CONTROL_MODE_HDMI);
 
                 ret = wait_for(!(HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
-                                 VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE), 1);
+                                 VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE), 1000);
                 WARN_ONCE(ret, "Timeout waiting for "
                           "!VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE\n");
         }
@@ -420,9 +560,10 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
                            HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) |
                            VC4_HDMI_SCHEDULER_CONTROL_VERT_ALWAYS_KEEPOUT);
 
-                /* XXX: Set HDMI_RAM_PACKET_CONFIG (1 << 16) and set
-                 * up the infoframe.
-                 */
+                HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG,
+                           VC4_HDMI_RAM_PACKET_ENABLE);
+
+                vc4_hdmi_set_infoframes(encoder);
 
                 drift = HDMI_READ(VC4_HDMI_FIFO_CTL);
                 drift &= VC4_HDMI_FIFO_VALID_WRITE_MASK;
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index 160942a9180e..1aa44c2db556 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -175,6 +175,8 @@
 # define PV_CONTROL_CLR_AT_START                BIT(14)
 # define PV_CONTROL_TRIGGER_UNDERFLOW           BIT(13)
 # define PV_CONTROL_WAIT_HSTART                 BIT(12)
+# define PV_CONTROL_PIXEL_REP_MASK              VC4_MASK(5, 4)
+# define PV_CONTROL_PIXEL_REP_SHIFT             4
 # define PV_CONTROL_CLK_SELECT_DSI_VEC          0
 # define PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI     1
 # define PV_CONTROL_CLK_SELECT_MASK             VC4_MASK(3, 2)
@@ -183,6 +185,9 @@
 # define PV_CONTROL_EN                          BIT(0)
 
 #define PV_V_CONTROL                            0x04
+# define PV_VCONTROL_ODD_DELAY_MASK             VC4_MASK(22, 6)
+# define PV_VCONTROL_ODD_DELAY_SHIFT            6
+# define PV_VCONTROL_ODD_FIRST                  BIT(5)
 # define PV_VCONTROL_INTERLACE                  BIT(4)
 # define PV_VCONTROL_CONTINUOUS                 BIT(1)
 # define PV_VCONTROL_VIDEN                      BIT(0)
@@ -438,6 +443,8 @@
 #define VC4_HDMI_RAM_PACKET_CONFIG              0x0a0
 # define VC4_HDMI_RAM_PACKET_ENABLE             BIT(16)
 
+#define VC4_HDMI_RAM_PACKET_STATUS              0x0a4
+
 #define VC4_HDMI_HORZA                          0x0c4
 # define VC4_HDMI_HORZA_VPOS                    BIT(14)
 # define VC4_HDMI_HORZA_HPOS                    BIT(13)
@@ -499,6 +506,9 @@
 
 #define VC4_HDMI_TX_PHY_RESET_CTL               0x2c0
 
+#define VC4_HDMI_GCP_0                          0x400
+#define VC4_HDMI_PACKET_STRIDE                  0x24
+
 #define VC4_HD_M_CTL                            0x00c
 # define VC4_HD_M_REGISTER_FILE_STANDBY         (3 << 6)
 # define VC4_HD_M_RAM_STANDBY                   (3 << 4)
@@ -528,10 +538,17 @@
 # define VC4_HD_CSC_CTL_MODE_SHIFT              2
 # define VC4_HD_CSC_CTL_MODE_RGB_TO_SD_YPRPB    0
 # define VC4_HD_CSC_CTL_MODE_RGB_TO_HD_YPRPB    1
-# define VC4_HD_CSC_CTL_MODE_CUSTOM             2
+# define VC4_HD_CSC_CTL_MODE_CUSTOM             3
 # define VC4_HD_CSC_CTL_RGB2YCC                 BIT(1)
 # define VC4_HD_CSC_CTL_ENABLE                  BIT(0)
 
+#define VC4_HD_CSC_12_11                        0x044
+#define VC4_HD_CSC_14_13                        0x048
+#define VC4_HD_CSC_22_21                        0x04c
+#define VC4_HD_CSC_24_23                        0x050
+#define VC4_HD_CSC_32_31                        0x054
+#define VC4_HD_CSC_34_33                        0x058
+
 #define VC4_HD_FRAME_COUNT                      0x068
 
 /* HVS display list information. */
diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c
index 0f12418725e5..08886a309757 100644
--- a/drivers/gpu/drm/vc4/vc4_render_cl.c
+++ b/drivers/gpu/drm/vc4/vc4_render_cl.c
@@ -45,6 +45,8 @@ struct vc4_rcl_setup {
 
         struct drm_gem_cma_object *rcl;
         u32 next_offset;
+
+        u32 next_write_bo_index;
 };
 
 static inline void rcl_u8(struct vc4_rcl_setup *setup, u8 val)
@@ -407,6 +409,8 @@ static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
         if (!*obj)
                 return -EINVAL;
 
+        exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
+
         if (surf->offset & 0xf) {
                 DRM_ERROR("MSAA write must be 16b aligned.\n");
                 return -EINVAL;
@@ -417,7 +421,8 @@ static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
 
 static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
                                  struct drm_gem_cma_object **obj,
-                                 struct drm_vc4_submit_rcl_surface *surf)
+                                 struct drm_vc4_submit_rcl_surface *surf,
+                                 bool is_write)
 {
         uint8_t tiling = VC4_GET_FIELD(surf->bits,
                                        VC4_LOADSTORE_TILE_BUFFER_TILING);
@@ -440,6 +445,9 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
         if (!*obj)
                 return -EINVAL;
 
+        if (is_write)
+                exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
+
         if (surf->flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
                 if (surf == &exec->args->zs_write) {
                         DRM_ERROR("general zs write may not be a full-res.\n");
@@ -542,6 +550,8 @@ vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
         if (!*obj)
                 return -EINVAL;
 
+        exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
+
         if (tiling > VC4_TILING_FORMAT_LT) {
                 DRM_ERROR("Bad tiling format\n");
                 return -EINVAL;
@@ -599,15 +609,18 @@ int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
         if (ret)
                 return ret;
 
-        ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read);
+        ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read,
+                                    false);
         if (ret)
                 return ret;
 
-        ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read);
+        ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read,
+                                    false);
         if (ret)
                 return ret;
 
-        ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write);
+        ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write,
+                                    true);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 9ce1d0adf882..26503e307438 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -267,6 +267,9 @@ validate_indexed_prim_list(VALIDATE_ARGS)
         if (!ib)
                 return -EINVAL;
 
+        exec->bin_dep_seqno = max(exec->bin_dep_seqno,
+                                  to_vc4_bo(&ib->base)->write_seqno);
+
         if (offset > ib->base.size ||
             (ib->base.size - offset) / index_size < length) {
                 DRM_ERROR("IB access overflow (%d + %d*%d > %zd)\n",
@@ -555,8 +558,7 @@ static bool
 reloc_tex(struct vc4_exec_info *exec,
           void *uniform_data_u,
           struct vc4_texture_sample_info *sample,
-          uint32_t texture_handle_index)
-
+          uint32_t texture_handle_index, bool is_cs)
 {
         struct drm_gem_cma_object *tex;
         uint32_t p0 = *(uint32_t *)(uniform_data_u + sample->p_offset[0]);
@@ -714,6 +716,11 @@ reloc_tex(struct vc4_exec_info *exec,
 
         *validated_p0 = tex->paddr + p0;
 
+        if (is_cs) {
+                exec->bin_dep_seqno = max(exec->bin_dep_seqno,
+                                          to_vc4_bo(&tex->base)->write_seqno);
+        }
+
         return true;
  fail:
         DRM_INFO("Texture p0 at %d: 0x%08x\n", sample->p_offset[0], p0);
@@ -835,7 +842,8 @@ validate_gl_shader_rec(struct drm_device *dev,
                         if (!reloc_tex(exec,
                                        uniform_data_u,
                                        &validated_shader->texture_samples[tex],
-                                       texture_handles_u[tex])) {
+                                       texture_handles_u[tex],
+                                       i == 2)) {
                                 return -EINVAL;
                         }
                 }
@@ -867,6 +875,9 @@ validate_gl_shader_rec(struct drm_device *dev,
                 uint32_t stride = *(uint8_t *)(pkt_u + o + 5);
                 uint32_t max_index;
 
+                exec->bin_dep_seqno = max(exec->bin_dep_seqno,
+                                          to_vc4_bo(&vbo->base)->write_seqno);
+
                 if (state->addr & 0x8)
                         stride |= (*(uint32_t *)(pkt_u + 100 + i * 4)) & ~0xff;
 
diff --git a/drivers/gpu/drm/via/via_dmablit.c b/drivers/gpu/drm/via/via_dmablit.c
index 7e2a12c4fed2..1a3ad769f8c8 100644
--- a/drivers/gpu/drm/via/via_dmablit.c
+++ b/drivers/gpu/drm/via/via_dmablit.c
@@ -241,8 +241,8 @@ via_lock_all_dma_pages(drm_via_sg_info_t *vsg,  drm_via_dmablit_t *xfer)
         down_read(&current->mm->mmap_sem);
         ret = get_user_pages((unsigned long)xfer->mem_addr,
                              vsg->num_pages,
-                             (vsg->direction == DMA_FROM_DEVICE),
-                             0, vsg->pages, NULL);
+                             (vsg->direction == DMA_FROM_DEVICE) ? FOLL_WRITE : 0,
+                             vsg->pages, NULL);
 
         up_read(&current->mm->mmap_sem);
         if (ret != vsg->num_pages) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index e8ae3dc476d1..18061a4bc2f2 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -241,15 +241,15 @@ static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
                               void *ptr);
 
 MODULE_PARM_DESC(enable_fbdev, "Enable vmwgfx fbdev");
-module_param_named(enable_fbdev, enable_fbdev, int, 0600);
+module_param_named(enable_fbdev, enable_fbdev, int, S_IRUSR | S_IWUSR);
 MODULE_PARM_DESC(force_dma_api, "Force using the DMA API for TTM pages");
-module_param_named(force_dma_api, vmw_force_iommu, int, 0600);
+module_param_named(force_dma_api, vmw_force_iommu, int, S_IRUSR | S_IWUSR);
 MODULE_PARM_DESC(restrict_iommu, "Try to limit IOMMU usage for TTM pages");
-module_param_named(restrict_iommu, vmw_restrict_iommu, int, 0600);
+module_param_named(restrict_iommu, vmw_restrict_iommu, int, S_IRUSR | S_IWUSR);
 MODULE_PARM_DESC(force_coherent, "Force coherent TTM pages");
-module_param_named(force_coherent, vmw_force_coherent, int, 0600);
+module_param_named(force_coherent, vmw_force_coherent, int, S_IRUSR | S_IWUSR);
 MODULE_PARM_DESC(restrict_dma_mask, "Restrict DMA mask to 44 bits with IOMMU");
-module_param_named(restrict_dma_mask, vmw_restrict_dma_mask, int, 0600);
+module_param_named(restrict_dma_mask, vmw_restrict_dma_mask, int, S_IRUSR | S_IWUSR);
 MODULE_PARM_DESC(assume_16bpp, "Assume 16-bpp when filtering modes");
 module_param_named(assume_16bpp, vmw_assume_16bpp, int, 0600);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 070d750af16d..1e59a486bba8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -43,7 +43,7 @@
 
 #define VMWGFX_DRIVER_DATE "20160210"
 #define VMWGFX_DRIVER_MAJOR 2
-#define VMWGFX_DRIVER_MINOR 10
+#define VMWGFX_DRIVER_MINOR 11
 #define VMWGFX_DRIVER_PATCHLEVEL 0
 #define VMWGFX_FILE_PAGE_OFFSET 0x00100000
 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index dc5beff2b4aa..c7b53d987f06 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -35,17 +35,37 @@
 #define VMW_RES_HT_ORDER 12
 
 /**
+ * enum vmw_resource_relocation_type - Relocation type for resources
+ *
+ * @vmw_res_rel_normal: Traditional relocation. The resource id in the
+ * command stream is replaced with the actual id after validation.
+ * @vmw_res_rel_nop: NOP relocation. The command is unconditionally replaced
+ * with a NOP.
+ * @vmw_res_rel_cond_nop: Conditional NOP relocation. If the resource id
+ * after validation is -1, the command is replaced with a NOP. Otherwise no
+ * action.
+ */
+enum vmw_resource_relocation_type {
+        vmw_res_rel_normal,
+        vmw_res_rel_nop,
+        vmw_res_rel_cond_nop,
+        vmw_res_rel_max
+};
+
+/**
  * struct vmw_resource_relocation - Relocation info for resources
  *
  * @head: List head for the software context's relocation list.
  * @res: Non-ref-counted pointer to the resource.
- * @offset: Offset of 4 byte entries into the command buffer where the
+ * @offset: Offset of single byte entries into the command buffer where the
  * id that needs fixup is located.
+ * @rel_type: Type of relocation.
  */
 struct vmw_resource_relocation {
         struct list_head head;
         const struct vmw_resource *res;
-        unsigned long offset;
+        u32 offset:29;
+        enum vmw_resource_relocation_type rel_type:3;
 };
 
 /**
@@ -109,7 +129,18 @@ static int vmw_bo_to_validate_list(struct vmw_sw_context *sw_context,
                                    struct vmw_dma_buffer *vbo,
                                    bool validate_as_mob,
                                    uint32_t *p_val_node);
-
+/**
+ * vmw_ptr_diff - Compute the offset from a to b in bytes
+ *
+ * @a: A starting pointer.
+ * @b: A pointer offset in the same address space.
+ *
+ * Returns: The offset in bytes between the two pointers.
+ */
+static size_t vmw_ptr_diff(void *a, void *b)
+{
+        return (unsigned long) b - (unsigned long) a;
+}
 
 /**
  * vmw_resources_unreserve - unreserve resources previously reserved for
@@ -409,11 +440,14 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
  * @list: Pointer to head of relocation list.
  * @res: The resource.
  * @offset: Offset into the command buffer currently being parsed where the
- * id that needs fixup is located. Granularity is 4 bytes.
+ * id that needs fixup is located. Granularity is one byte.
+ * @rel_type: Relocation type.
  */
 static int vmw_resource_relocation_add(struct list_head *list,
                                        const struct vmw_resource *res,
-                                       unsigned long offset)
+                                       unsigned long offset,
+                                       enum vmw_resource_relocation_type
+                                       rel_type)
 {
         struct vmw_resource_relocation *rel;
 
@@ -425,6 +459,7 @@ static int vmw_resource_relocation_add(struct list_head *list,
 
         rel->res = res;
         rel->offset = offset;
+        rel->rel_type = rel_type;
         list_add_tail(&rel->head, list);
 
         return 0;
@@ -459,11 +494,24 @@ static void vmw_resource_relocations_apply(uint32_t *cb,
 {
         struct vmw_resource_relocation *rel;
 
+        /* Validate the struct vmw_resource_relocation member size */
+        BUILD_BUG_ON(SVGA_CB_MAX_SIZE >= (1 << 29));
+        BUILD_BUG_ON(vmw_res_rel_max >= (1 << 3));
+
         list_for_each_entry(rel, list, head) {
-                if (likely(rel->res != NULL))
-                        cb[rel->offset] = rel->res->id;
-                else
-                        cb[rel->offset] = SVGA_3D_CMD_NOP;
+                u32 *addr = (u32 *)((unsigned long) cb + rel->offset);
+                switch (rel->rel_type) {
+                case vmw_res_rel_normal:
+                        *addr = rel->res->id;
+                        break;
+                case vmw_res_rel_nop:
+                        *addr = SVGA_3D_CMD_NOP;
+                        break;
+                default:
+                        if (rel->res->id == -1)
+                                *addr = SVGA_3D_CMD_NOP;
+                        break;
+                }
         }
 }
 
@@ -655,7 +703,9 @@ static int vmw_cmd_res_reloc_add(struct vmw_private *dev_priv,
         *p_val = NULL;
         ret = vmw_resource_relocation_add(&sw_context->res_relocations,
                                           res,
-                                          id_loc - sw_context->buf_start);
+                                          vmw_ptr_diff(sw_context->buf_start,
+                                                       id_loc),
+                                          vmw_res_rel_normal);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -721,7 +771,8 @@ vmw_cmd_res_check(struct vmw_private *dev_priv,
 
                 return vmw_resource_relocation_add
                         (&sw_context->res_relocations, res,
-                         id_loc - sw_context->buf_start);
+                         vmw_ptr_diff(sw_context->buf_start, id_loc),
+                         vmw_res_rel_normal);
         }
 
         ret = vmw_user_resource_lookup_handle(dev_priv,
@@ -2143,10 +2194,10 @@ static int vmw_cmd_shader_define(struct vmw_private *dev_priv,
                 return ret;
 
         return vmw_resource_relocation_add(&sw_context->res_relocations,
-                                           NULL, &cmd->header.id -
-                                           sw_context->buf_start);
-
-        return 0;
+                                           NULL,
+                                           vmw_ptr_diff(sw_context->buf_start,
+                                                        &cmd->header.id),
+                                           vmw_res_rel_nop);
 }
 
 /**
@@ -2188,10 +2239,10 @@ static int vmw_cmd_shader_destroy(struct vmw_private *dev_priv,
                 return ret;
 
         return vmw_resource_relocation_add(&sw_context->res_relocations,
-                                           NULL, &cmd->header.id -
-                                           sw_context->buf_start);
-
-        return 0;
+                                           NULL,
+                                           vmw_ptr_diff(sw_context->buf_start,
+                                                        &cmd->header.id),
+                                           vmw_res_rel_nop);
 }
 
 /**
@@ -2848,8 +2899,7 @@ static int vmw_cmd_dx_cid_check(struct vmw_private *dev_priv,
  * @header: Pointer to the command header in the command stream.
  *
  * Check that the view exists, and if it was not created using this
- * command batch, make sure it's validated (present in the device) so that
- * the remove command will not confuse the device.
+ * command batch, conditionally make this command a NOP.
  */
 static int vmw_cmd_dx_view_remove(struct vmw_private *dev_priv,
                                   struct vmw_sw_context *sw_context,
@@ -2877,10 +2927,16 @@ static int vmw_cmd_dx_view_remove(struct vmw_private *dev_priv,
                 return ret;
 
         /*
-         * Add view to the validate list iff it was not created using this
-         * command batch.
+         * If the view wasn't created during this command batch, it might
+         * have been removed due to a context swapout, so add a
+         * relocation to conditionally make this command a NOP to avoid
+         * device errors.
          */
-        return vmw_view_res_val_add(sw_context, view);
+        return vmw_resource_relocation_add(&sw_context->res_relocations,
+                                           view,
+                                           vmw_ptr_diff(sw_context->buf_start,
+                                                        &cmd->header.id),
+                                           vmw_res_rel_cond_nop);
 }
 
 /**
@@ -3029,6 +3085,35 @@ static int vmw_cmd_dx_genmips(struct vmw_private *dev_priv,
                                    cmd->body.shaderResourceViewId);
 }
 
+/**
+ * vmw_cmd_dx_transfer_from_buffer -
+ * Validate an SVGA_3D_CMD_DX_TRANSFER_FROM_BUFFER command
+ *
+ * @dev_priv: Pointer to a device private struct.
+ * @sw_context: The software context being used for this batch.
+ * @header: Pointer to the command header in the command stream.
+ */
+static int vmw_cmd_dx_transfer_from_buffer(struct vmw_private *dev_priv,
+                                           struct vmw_sw_context *sw_context,
+                                           SVGA3dCmdHeader *header)
+{
+        struct {
+                SVGA3dCmdHeader header;
+                SVGA3dCmdDXTransferFromBuffer body;
+        } *cmd = container_of(header, typeof(*cmd), header);
+        int ret;
+
+        ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+                                user_surface_converter,
+                                &cmd->body.srcSid, NULL);
+        if (ret != 0)
+                return ret;
+
+        return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+                                 user_surface_converter,
+                                 &cmd->body.destSid, NULL);
+}
+
 static int vmw_cmd_check_not_3d(struct vmw_private *dev_priv,
                                 struct vmw_sw_context *sw_context,
                                 void *buf, uint32_t *size)
@@ -3379,6 +3464,9 @@ static const struct vmw_cmd_entry vmw_cmd_entries[SVGA_3D_CMD_MAX] = {
                     &vmw_cmd_buffer_copy_check, true, false, true),
         VMW_CMD_DEF(SVGA_3D_CMD_DX_PRED_COPY_REGION,
                     &vmw_cmd_pred_copy_check, true, false, true),
+        VMW_CMD_DEF(SVGA_3D_CMD_DX_TRANSFER_FROM_BUFFER,
+                    &vmw_cmd_dx_transfer_from_buffer,
+                    true, false, true),
 };
 
 static int vmw_cmd_check(struct vmw_private *dev_priv,
@@ -3848,14 +3936,14 @@ static void *vmw_execbuf_cmdbuf(struct vmw_private *dev_priv,
         int ret;
 
         *header = NULL;
-        if (!dev_priv->cman || kernel_commands)
-                return kernel_commands;
-
         if (command_size > SVGA_CB_MAX_SIZE) {
                 DRM_ERROR("Command buffer is too large.\n");
                 return ERR_PTR(-EINVAL);
         }
 
+        if (!dev_priv->cman || kernel_commands)
+                return kernel_commands;
+
         /* If possible, add a little space for fencing. */
         cmdbuf_size = command_size + 512;
         cmdbuf_size = min_t(size_t, cmdbuf_size, SVGA_CB_MAX_SIZE);
@@ -4232,9 +4320,6 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
         ttm_bo_unref(&query_val.bo);
         ttm_bo_unref(&pinned_val.bo);
         vmw_dmabuf_unreference(&dev_priv->pinned_bo);
-        DRM_INFO("Dummy query bo pin count: %d\n",
-                 dev_priv->dummy_query_bo->pin_count);
-
 out_unlock:
         return;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index c2a721a8cef9..b445ce9b9757 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -324,7 +324,7 @@ static void vmw_hw_surface_destroy(struct vmw_resource *res)
         if (res->id != -1) {
 
                 cmd = vmw_fifo_reserve(dev_priv, vmw_surface_destroy_size());
-                if (unlikely(cmd == NULL)) {
+                if (unlikely(!cmd)) {
                         DRM_ERROR("Failed reserving FIFO space for surface "
                                   "destruction.\n");
                         return;
@@ -397,7 +397,7 @@ static int vmw_legacy_srf_create(struct vmw_resource *res)
 
         submit_size = vmw_surface_define_size(srf);
         cmd = vmw_fifo_reserve(dev_priv, submit_size);
-        if (unlikely(cmd == NULL)) {
+        if (unlikely(!cmd)) {
                 DRM_ERROR("Failed reserving FIFO space for surface "
                           "creation.\n");
                 ret = -ENOMEM;
@@ -446,11 +446,10 @@ static int vmw_legacy_srf_dma(struct vmw_resource *res,
         uint8_t *cmd;
         struct vmw_private *dev_priv = res->dev_priv;
 
-        BUG_ON(val_buf->bo == NULL);
-
+        BUG_ON(!val_buf->bo);
         submit_size = vmw_surface_dma_size(srf);
         cmd = vmw_fifo_reserve(dev_priv, submit_size);
-        if (unlikely(cmd == NULL)) {
+        if (unlikely(!cmd)) {
                 DRM_ERROR("Failed reserving FIFO space for surface "
                           "DMA.\n");
                 return -ENOMEM;
@@ -538,7 +537,7 @@ static int vmw_legacy_srf_destroy(struct vmw_resource *res)
 
         submit_size = vmw_surface_destroy_size();
         cmd = vmw_fifo_reserve(dev_priv, submit_size);
-        if (unlikely(cmd == NULL)) {
+        if (unlikely(!cmd)) {
                 DRM_ERROR("Failed reserving FIFO space for surface "
                           "eviction.\n");
                 return -ENOMEM;
@@ -578,7 +577,7 @@ static int vmw_surface_init(struct vmw_private *dev_priv,
         int ret;
         struct vmw_resource *res = &srf->res;
 
-        BUG_ON(res_free == NULL);
+        BUG_ON(!res_free);
         if (!dev_priv->has_mob)
                 vmw_fifo_resource_inc(dev_priv);
         ret = vmw_resource_init(dev_priv, res, true, res_free,
@@ -700,7 +699,6 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
         struct drm_vmw_surface_create_req *req = &arg->req;
         struct drm_vmw_surface_arg *rep = &arg->rep;
         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-        struct drm_vmw_size __user *user_sizes;
         int ret;
         int i, j;
         uint32_t cur_bo_offset;
@@ -748,7 +746,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
         }
 
         user_srf = kzalloc(sizeof(*user_srf), GFP_KERNEL);
-        if (unlikely(user_srf == NULL)) {
+        if (unlikely(!user_srf)) {
                 ret = -ENOMEM;
                 goto out_no_user_srf;
         }
@@ -763,29 +761,21 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
         memcpy(srf->mip_levels, req->mip_levels, sizeof(srf->mip_levels));
         srf->num_sizes = num_sizes;
         user_srf->size = size;
-
-        srf->sizes = kmalloc(srf->num_sizes * sizeof(*srf->sizes), GFP_KERNEL);
-        if (unlikely(srf->sizes == NULL)) {
-                ret = -ENOMEM;
+        srf->sizes = memdup_user((struct drm_vmw_size __user *)(unsigned long)
+                                 req->size_addr,
+                                 sizeof(*srf->sizes) * srf->num_sizes);
+        if (IS_ERR(srf->sizes)) {
+                ret = PTR_ERR(srf->sizes);
                 goto out_no_sizes;
         }
-        srf->offsets = kmalloc(srf->num_sizes * sizeof(*srf->offsets),
-                               GFP_KERNEL);
-        if (unlikely(srf->offsets == NULL)) {
+        srf->offsets = kmalloc_array(srf->num_sizes,
+                                     sizeof(*srf->offsets),
+                                     GFP_KERNEL);
+        if (unlikely(!srf->offsets)) {
                 ret = -ENOMEM;
                 goto out_no_offsets;
         }
 
-        user_sizes = (struct drm_vmw_size __user *)(unsigned long)
-            req->size_addr;
-
-        ret = copy_from_user(srf->sizes, user_sizes,
-                             srf->num_sizes * sizeof(*srf->sizes));
-        if (unlikely(ret != 0)) {
-                ret = -EFAULT;
-                goto out_no_copy;
-        }
-
         srf->base_size = *srf->sizes;
         srf->autogen_filter = SVGA3D_TEX_FILTER_NONE;
         srf->multisample_count = 0;
@@ -923,7 +913,7 @@ vmw_surface_handle_reference(struct vmw_private *dev_priv,
 
         ret = -EINVAL;
         base = ttm_base_object_lookup_for_ref(dev_priv->tdev, handle);
-        if (unlikely(base == NULL)) {
+        if (unlikely(!base)) {
                 DRM_ERROR("Could not find surface to reference.\n");
                 goto out_no_lookup;
         }
@@ -1069,7 +1059,7 @@ static int vmw_gb_surface_create(struct vmw_resource *res)
 
         cmd = vmw_fifo_reserve(dev_priv, submit_len);
         cmd2 = (typeof(cmd2))cmd;
-        if (unlikely(cmd == NULL)) {
+        if (unlikely(!cmd)) {
                 DRM_ERROR("Failed reserving FIFO space for surface "
                           "creation.\n");
                 ret = -ENOMEM;
@@ -1135,7 +1125,7 @@ static int vmw_gb_surface_bind(struct vmw_resource *res,
         submit_size = sizeof(*cmd1) + (res->backup_dirty ? sizeof(*cmd2) : 0);
 
         cmd1 = vmw_fifo_reserve(dev_priv, submit_size);
-        if (unlikely(cmd1 == NULL)) {
+        if (unlikely(!cmd1)) {
                 DRM_ERROR("Failed reserving FIFO space for surface "
                           "binding.\n");
                 return -ENOMEM;
@@ -1185,7 +1175,7 @@ static int vmw_gb_surface_unbind(struct vmw_resource *res,
 
         submit_size = sizeof(*cmd3) + (readback ? sizeof(*cmd1) : sizeof(*cmd2));
         cmd = vmw_fifo_reserve(dev_priv, submit_size);
-        if (unlikely(cmd == NULL)) {
+        if (unlikely(!cmd)) {
                 DRM_ERROR("Failed reserving FIFO space for surface "
                           "unbinding.\n");
                 return -ENOMEM;
@@ -1244,7 +1234,7 @@ static int vmw_gb_surface_destroy(struct vmw_resource *res)
         vmw_binding_res_list_scrub(&res->binding_head);
 
         cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-        if (unlikely(cmd == NULL)) {
+        if (unlikely(!cmd)) {
                 DRM_ERROR("Failed reserving FIFO space for surface "
                           "destruction.\n");
                 mutex_unlock(&dev_priv->binding_mutex);
@@ -1410,7 +1400,7 @@ int vmw_gb_surface_reference_ioctl(struct drm_device *dev, void *data,
 
         user_srf = container_of(base, struct vmw_user_surface, prime.base);
         srf = &user_srf->srf;
-        if (srf->res.backup == NULL) {
+        if (!srf->res.backup) {
                 DRM_ERROR("Shared GB surface is missing a backup buffer.\n");
                 goto out_bad_resource;
         }
@@ -1524,7 +1514,7 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
         }
 
         user_srf = kzalloc(sizeof(*user_srf), GFP_KERNEL);
-        if (unlikely(user_srf == NULL)) {
+        if (unlikely(!user_srf)) {
                 ret = -ENOMEM;
                 goto out_no_user_srf;
         }