1 files changed, 290 insertions, 174 deletions

diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 72f41aaa71ff..0cbc0e6402b4 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -1468,9 +1468,12 @@ static void i9xx_update_wm(struct drm_device *dev)
         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
         crtc = intel_get_crtc_for_plane(dev, 0);
         if (crtc->enabled && crtc->fb) {
+                int cpp = crtc->fb->bits_per_pixel / 8;
+                if (IS_GEN2(dev))
+                        cpp = 4;
+
                 planea_wm = intel_calculate_wm(crtc->mode.clock,
-                                               wm_info, fifo_size,
-                                               crtc->fb->bits_per_pixel / 8,
+                                               wm_info, fifo_size, cpp,
                                                latency_ns);
                 enabled = crtc;
         } else
@@ -1479,9 +1482,12 @@ static void i9xx_update_wm(struct drm_device *dev)
         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
         crtc = intel_get_crtc_for_plane(dev, 1);
         if (crtc->enabled && crtc->fb) {
+                int cpp = crtc->fb->bits_per_pixel / 8;
+                if (IS_GEN2(dev))
+                        cpp = 4;
+
                 planeb_wm = intel_calculate_wm(crtc->mode.clock,
-                                               wm_info, fifo_size,
-                                               crtc->fb->bits_per_pixel / 8,
+                                               wm_info, fifo_size, cpp,
                                                latency_ns);
                 if (enabled == NULL)
                         enabled = crtc;
@@ -1571,8 +1577,7 @@ static void i830_update_wm(struct drm_device *dev)
 
         planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
                                        dev_priv->display.get_fifo_size(dev, 0),
-                                       crtc->fb->bits_per_pixel / 8,
-                                       latency_ns);
+                                       4, latency_ns);
         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
         fwater_lo |= (3<<8) | planea_wm;
 
@@ -2323,7 +2328,7 @@ void gen6_set_rps(struct drm_device *dev, u8 val)
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 limits = gen6_rps_limits(dev_priv, &val);
 
-        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
         WARN_ON(val > dev_priv->rps.max_delay);
         WARN_ON(val < dev_priv->rps.min_delay);
 
@@ -2404,12 +2409,12 @@ static void gen6_enable_rps(struct drm_device *dev)
         struct intel_ring_buffer *ring;
         u32 rp_state_cap;
         u32 gt_perf_status;
-        u32 pcu_mbox, rc6_mask = 0;
+        u32 rc6vids, pcu_mbox, rc6_mask = 0;
         u32 gtfifodbg;
         int rc6_mode;
-        int i;
+        int i, ret;
 
-        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
 
         /* Here begins a magic sequence of register writes to enable
          * auto-downclocking.
@@ -2503,30 +2508,16 @@ static void gen6_enable_rps(struct drm_device *dev)
                    GEN6_RP_UP_BUSY_AVG |
                    (IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
 
-        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                     500))
-                DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
-
-        I915_WRITE(GEN6_PCODE_DATA, 0);
-        I915_WRITE(GEN6_PCODE_MAILBOX,
-                   GEN6_PCODE_READY |
-                   GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
-        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                     500))
-                DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
-
-        /* Check for overclock support */
-        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                     500))
-                DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
-        I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
-        pcu_mbox = I915_READ(GEN6_PCODE_DATA);
-        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                     500))
-                DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
-        if (pcu_mbox & (1<<31)) { /* OC supported */
-                dev_priv->rps.max_delay = pcu_mbox & 0xff;
-                DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
+        ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
+        if (!ret) {
+                pcu_mbox = 0;
+                ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
+                if (ret && pcu_mbox & (1<<31)) { /* OC supported */
+                        dev_priv->rps.max_delay = pcu_mbox & 0xff;
+                        DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
+                }
+        } else {
+                DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
         }
 
         gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
@@ -2540,6 +2531,20 @@ static void gen6_enable_rps(struct drm_device *dev)
         /* enable all PM interrupts */
         I915_WRITE(GEN6_PMINTRMSK, 0);
 
+        rc6vids = 0;
+        ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
+        if (IS_GEN6(dev) && ret) {
+                DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
+        } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
+                DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
+                          GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
+                rc6vids &= 0xffff00;
+                rc6vids |= GEN6_ENCODE_RC6_VID(450);
+                ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
+                if (ret)
+                        DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
+        }
+
         gen6_gt_force_wake_put(dev_priv);
 }
 
@@ -2550,7 +2555,7 @@ static void gen6_update_ring_freq(struct drm_device *dev)
         int gpu_freq, ia_freq, max_ia_freq;
         int scaling_factor = 180;
 
-        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
 
         max_ia_freq = cpufreq_quick_get_max(0);
         /*
@@ -2581,17 +2586,11 @@ static void gen6_update_ring_freq(struct drm_device *dev)
                 else
                         ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
                 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
+                ia_freq <<= GEN6_PCODE_FREQ_IA_RATIO_SHIFT;
 
-                I915_WRITE(GEN6_PCODE_DATA,
-                           (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
-                           gpu_freq);
-                I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
-                           GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
-                if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
-                              GEN6_PCODE_READY) == 0, 10)) {
-                        DRM_ERROR("pcode write of freq table timed out\n");
-                        continue;
-                }
+                sandybridge_pcode_write(dev_priv,
+                                        GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
+                                        ia_freq | gpu_freq);
         }
 }
 
@@ -2599,16 +2598,16 @@ void ironlake_teardown_rc6(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (dev_priv->renderctx) {
-                i915_gem_object_unpin(dev_priv->renderctx);
-                drm_gem_object_unreference(&dev_priv->renderctx->base);
-                dev_priv->renderctx = NULL;
+        if (dev_priv->ips.renderctx) {
+                i915_gem_object_unpin(dev_priv->ips.renderctx);
+                drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
+                dev_priv->ips.renderctx = NULL;
         }
 
-        if (dev_priv->pwrctx) {
-                i915_gem_object_unpin(dev_priv->pwrctx);
-                drm_gem_object_unreference(&dev_priv->pwrctx->base);
-                dev_priv->pwrctx = NULL;
+        if (dev_priv->ips.pwrctx) {
+                i915_gem_object_unpin(dev_priv->ips.pwrctx);
+                drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
+                dev_priv->ips.pwrctx = NULL;
         }
 }
 
@@ -2634,14 +2633,14 @@ static int ironlake_setup_rc6(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (dev_priv->renderctx == NULL)
-                dev_priv->renderctx = intel_alloc_context_page(dev);
-        if (!dev_priv->renderctx)
+        if (dev_priv->ips.renderctx == NULL)
+                dev_priv->ips.renderctx = intel_alloc_context_page(dev);
+        if (!dev_priv->ips.renderctx)
                 return -ENOMEM;
 
-        if (dev_priv->pwrctx == NULL)
-                dev_priv->pwrctx = intel_alloc_context_page(dev);
-        if (!dev_priv->pwrctx) {
+        if (dev_priv->ips.pwrctx == NULL)
+                dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
+        if (!dev_priv->ips.pwrctx) {
                 ironlake_teardown_rc6(dev);
                 return -ENOMEM;
         }
@@ -2679,7 +2678,7 @@ static void ironlake_enable_rc6(struct drm_device *dev)
 
         intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
         intel_ring_emit(ring, MI_SET_CONTEXT);
-        intel_ring_emit(ring, dev_priv->renderctx->gtt_offset |
+        intel_ring_emit(ring, dev_priv->ips.renderctx->gtt_offset |
                         MI_MM_SPACE_GTT |
                         MI_SAVE_EXT_STATE_EN |
                         MI_RESTORE_EXT_STATE_EN |
@@ -2701,7 +2700,7 @@ static void ironlake_enable_rc6(struct drm_device *dev)
                 return;
         }
 
-        I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN);
+        I915_WRITE(PWRCTXA, dev_priv->ips.pwrctx->gtt_offset | PWRCTX_EN);
         I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
 }
 
@@ -3310,37 +3309,72 @@ static void intel_init_emon(struct drm_device *dev)
 
 void intel_disable_gt_powersave(struct drm_device *dev)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
         if (IS_IRONLAKE_M(dev)) {
                 ironlake_disable_drps(dev);
                 ironlake_disable_rc6(dev);
         } else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
+                cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
+                mutex_lock(&dev_priv->rps.hw_lock);
                 gen6_disable_rps(dev);
+                mutex_unlock(&dev_priv->rps.hw_lock);
         }
 }
 
+static void intel_gen6_powersave_work(struct work_struct *work)
+{
+        struct drm_i915_private *dev_priv =
+                container_of(work, struct drm_i915_private,
+                             rps.delayed_resume_work.work);
+        struct drm_device *dev = dev_priv->dev;
+
+        mutex_lock(&dev_priv->rps.hw_lock);
+        gen6_enable_rps(dev);
+        gen6_update_ring_freq(dev);
+        mutex_unlock(&dev_priv->rps.hw_lock);
+}
+
 void intel_enable_gt_powersave(struct drm_device *dev)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
         if (IS_IRONLAKE_M(dev)) {
                 ironlake_enable_drps(dev);
                 ironlake_enable_rc6(dev);
                 intel_init_emon(dev);
         } else if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
-                gen6_enable_rps(dev);
-                gen6_update_ring_freq(dev);
+                /*
+                 * PCU communication is slow and this doesn't need to be
+                 * done at any specific time, so do this out of our fast path
+                 * to make resume and init faster.
+                 */
+                schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
+                                      round_jiffies_up_relative(HZ));
         }
 }
 
+static void ibx_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /*
+         * On Ibex Peak and Cougar Point, we need to disable clock
+         * gating for the panel power sequencer or it will fail to
+         * start up when no ports are active.
+         */
+        I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+}
+
 static void ironlake_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+        uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
         /* Required for FBC */
-        dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
-                DPFCRUNIT_CLOCK_GATE_DISABLE |
-                DPFDUNIT_CLOCK_GATE_DISABLE;
-        /* Required for CxSR */
-        dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
+        dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
+                   ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
+                   ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
 
         I915_WRITE(PCH_3DCGDIS0,
                    MARIUNIT_CLOCK_GATE_DISABLE |
@@ -3348,8 +3382,6 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
         I915_WRITE(PCH_3DCGDIS1,
                    VFMUNIT_CLOCK_GATE_DISABLE);
 
-        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
-
         /*
          * According to the spec the following bits should be set in
          * order to enable memory self-refresh
@@ -3360,9 +3392,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
         I915_WRITE(ILK_DISPLAY_CHICKEN2,
                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
-        I915_WRITE(ILK_DSPCLK_GATE,
-                   (I915_READ(ILK_DSPCLK_GATE) |
-                    ILK_DPARB_CLK_GATE));
+        dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
         I915_WRITE(DISP_ARB_CTL,
                    (I915_READ(DISP_ARB_CTL) |
                     DISP_FBC_WM_DIS));
@@ -3384,28 +3414,51 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
                            I915_READ(ILK_DISPLAY_CHICKEN2) |
                            ILK_DPARB_GATE);
-                I915_WRITE(ILK_DSPCLK_GATE,
-                           I915_READ(ILK_DSPCLK_GATE) |
-                           ILK_DPFC_DIS1 |
-                           ILK_DPFC_DIS2 |
-                           ILK_CLK_FBC);
         }
 
+        I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
+
         I915_WRITE(ILK_DISPLAY_CHICKEN2,
                    I915_READ(ILK_DISPLAY_CHICKEN2) |
                    ILK_ELPIN_409_SELECT);
         I915_WRITE(_3D_CHICKEN2,
                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
                    _3D_CHICKEN2_WM_READ_PIPELINED);
+
+        /* WaDisableRenderCachePipelinedFlush */
+        I915_WRITE(CACHE_MODE_0,
+                   _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
+
+        ibx_init_clock_gating(dev);
+}
+
+static void cpt_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe;
+
+        /*
+         * On Ibex Peak and Cougar Point, we need to disable clock
+         * gating for the panel power sequencer or it will fail to
+         * start up when no ports are active.
+         */
+        I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+        I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
+                   DPLS_EDP_PPS_FIX_DIS);
+        /* WADP0ClockGatingDisable */
+        for_each_pipe(pipe) {
+                I915_WRITE(TRANS_CHICKEN1(pipe),
+                           TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
+        }
 }
 
 static void gen6_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
-        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+        uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
-        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+        I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
 
         I915_WRITE(ILK_DISPLAY_CHICKEN2,
                    I915_READ(ILK_DISPLAY_CHICKEN2) |
@@ -3460,10 +3513,10 @@ static void gen6_init_clock_gating(struct drm_device *dev)
         I915_WRITE(ILK_DISPLAY_CHICKEN2,
                    I915_READ(ILK_DISPLAY_CHICKEN2) |
                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
-        I915_WRITE(ILK_DSPCLK_GATE,
-                   I915_READ(ILK_DSPCLK_GATE) |
-                   ILK_DPARB_CLK_GATE  |
-                   ILK_DPFD_CLK_GATE);
+        I915_WRITE(ILK_DSPCLK_GATE_D,
+                   I915_READ(ILK_DSPCLK_GATE_D) |
+                   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
+                   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 
         I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
                    GEN6_MBCTL_ENABLE_BOOT_FETCH);
@@ -3479,6 +3532,8 @@ static void gen6_init_clock_gating(struct drm_device *dev)
          * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
         I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
         I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
+
+        cpt_init_clock_gating(dev);
 }
 
 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
@@ -3497,9 +3552,6 @@ static void haswell_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
-        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
-
-        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
 
         I915_WRITE(WM3_LP_ILK, 0);
         I915_WRITE(WM2_LP_ILK, 0);
@@ -3510,12 +3562,6 @@ static void haswell_init_clock_gating(struct drm_device *dev)
          */
         I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 
-        I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
-
-        I915_WRITE(IVB_CHICKEN3,
-                   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
-                   CHICKEN3_DGMG_DONE_FIX_DISABLE);
-
         /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
@@ -3559,21 +3605,31 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
-        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
         uint32_t snpcr;
 
-        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
-
         I915_WRITE(WM3_LP_ILK, 0);
         I915_WRITE(WM2_LP_ILK, 0);
         I915_WRITE(WM1_LP_ILK, 0);
 
-        I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
+        I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
+
+        /* WaDisableEarlyCull */
+        I915_WRITE(_3D_CHICKEN3,
+                   _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
 
+        /* WaDisableBackToBackFlipFix */
         I915_WRITE(IVB_CHICKEN3,
                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
+        /* WaDisablePSDDualDispatchEnable */
+        if (IS_IVB_GT1(dev))
+                I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
+                           _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
+        else
+                I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
+                           _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
+
         /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
@@ -3582,7 +3638,18 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
         I915_WRITE(GEN7_L3CNTLREG1,
                         GEN7_WA_FOR_GEN7_L3_CONTROL);
         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
-                        GEN7_WA_L3_CHICKEN_MODE);
+                   GEN7_WA_L3_CHICKEN_MODE);
+        if (IS_IVB_GT1(dev))
+                I915_WRITE(GEN7_ROW_CHICKEN2,
+                           _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+        else
+                I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
+                           _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+
+
+        /* WaForceL3Serialization */
+        I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
+                   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
 
         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
          * gating disable must be set.  Failure to set it results in
@@ -3626,34 +3693,53 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
         snpcr &= ~GEN6_MBC_SNPCR_MASK;
         snpcr |= GEN6_MBC_SNPCR_MED;
         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
+
+        cpt_init_clock_gating(dev);
 }
 
 static void valleyview_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
-        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
-
-        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
 
         I915_WRITE(WM3_LP_ILK, 0);
         I915_WRITE(WM2_LP_ILK, 0);
         I915_WRITE(WM1_LP_ILK, 0);
 
-        I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
+        I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
+
+        /* WaDisableEarlyCull */
+        I915_WRITE(_3D_CHICKEN3,
+                   _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
 
+        /* WaDisableBackToBackFlipFix */
         I915_WRITE(IVB_CHICKEN3,
                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
+        I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
+                   _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
+
         /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 
         /* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
-        I915_WRITE(GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
+        I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS);
         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 
+        /* WaForceL3Serialization */
+        I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
+                   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
+
+        /* WaDisableDopClockGating */
+        I915_WRITE(GEN7_ROW_CHICKEN2,
+                   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+
+        /* WaForceL3Serialization */
+        I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
+                   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
+
         /* This is required by WaCatErrorRejectionIssue */
         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
@@ -3710,6 +3796,13 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
                    PIPEA_HLINE_INT_EN | PIPEA_VBLANK_INT_EN |
                    SPRITEB_FLIPDONE_INT_EN | SPRITEA_FLIPDONE_INT_EN |
                    PLANEA_FLIPDONE_INT_EN);
+
+        /*
+         * WaDisableVLVClockGating_VBIIssue
+         * Disable clock gating on th GCFG unit to prevent a delay
+         * in the reporting of vblank events.
+         */
+        I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 }
 
 static void g4x_init_clock_gating(struct drm_device *dev)
@@ -3728,6 +3821,10 @@ static void g4x_init_clock_gating(struct drm_device *dev)
         if (IS_GM45(dev))
                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
+
+        /* WaDisableRenderCachePipelinedFlush */
+        I915_WRITE(CACHE_MODE_0,
+                   _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
 }
 
 static void crestline_init_clock_gating(struct drm_device *dev)
@@ -3783,44 +3880,11 @@ static void i830_init_clock_gating(struct drm_device *dev)
         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
 }
 
-static void ibx_init_clock_gating(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        /*
-         * On Ibex Peak and Cougar Point, we need to disable clock
-         * gating for the panel power sequencer or it will fail to
-         * start up when no ports are active.
-         */
-        I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
-}
-
-static void cpt_init_clock_gating(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe;
-
-        /*
-         * On Ibex Peak and Cougar Point, we need to disable clock
-         * gating for the panel power sequencer or it will fail to
-         * start up when no ports are active.
-         */
-        I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
-        I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
-                   DPLS_EDP_PPS_FIX_DIS);
-        /* Without this, mode sets may fail silently on FDI */
-        for_each_pipe(pipe)
-                I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
-}
-
 void intel_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         dev_priv->display.init_clock_gating(dev);
-
-        if (dev_priv->display.init_pch_clock_gating)
-                dev_priv->display.init_pch_clock_gating(dev);
 }
 
 /* Starting with Haswell, we have different power wells for
@@ -3846,7 +3910,7 @@ void intel_init_power_wells(struct drm_device *dev)
 
                 if ((well & HSW_PWR_WELL_STATE) == 0) {
                         I915_WRITE(power_wells[i], well & HSW_PWR_WELL_ENABLE);
-                        if (wait_for(I915_READ(power_wells[i] & HSW_PWR_WELL_STATE), 20))
+                        if (wait_for((I915_READ(power_wells[i]) & HSW_PWR_WELL_STATE), 20))
                                 DRM_ERROR("Error enabling power well %lx\n", power_wells[i]);
                 }
         }
@@ -3884,11 +3948,6 @@ void intel_init_pm(struct drm_device *dev)
 
         /* For FIFO watermark updates */
         if (HAS_PCH_SPLIT(dev)) {
-                if (HAS_PCH_IBX(dev))
-                        dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
-                else if (HAS_PCH_CPT(dev))
-                        dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
-
                 if (IS_GEN5(dev)) {
                         if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
                                 dev_priv->display.update_wm = ironlake_update_wm;
@@ -3999,6 +4058,12 @@ static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
                 DRM_ERROR("GT thread status wait timed out\n");
 }
 
+static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE_NOTRACE(FORCEWAKE, 0);
+        POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
+}
+
 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 {
         u32 forcewake_ack;
@@ -4012,7 +4077,7 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
                             FORCEWAKE_ACK_TIMEOUT_MS))
                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
-        I915_WRITE_NOTRACE(FORCEWAKE, 1);
+        I915_WRITE_NOTRACE(FORCEWAKE, FORCEWAKE_KERNEL);
         POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
 
         if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
@@ -4022,6 +4087,12 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
         __gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
+static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
+        POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
+}
+
 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
 {
         u32 forcewake_ack;
@@ -4035,7 +4106,7 @@ static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
                             FORCEWAKE_ACK_TIMEOUT_MS))
                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
-        I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(1));
+        I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
         POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
 
         if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
@@ -4079,7 +4150,7 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 
 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
 {
-        I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(1));
+        I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
         /* gen6_gt_check_fifodbg doubles as the POSTING_READ */
         gen6_gt_check_fifodbg(dev_priv);
 }
@@ -4117,13 +4188,18 @@ int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
         return ret;
 }
 
+static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff));
+}
+
 static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
 {
         if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0,
                             FORCEWAKE_ACK_TIMEOUT_MS))
                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
-        I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(1));
+        I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 
         if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1),
                             FORCEWAKE_ACK_TIMEOUT_MS))
@@ -4134,49 +4210,89 @@ static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
 
 static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
 {
-        I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(1));
+        I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
         /* The below doubles as a POSTING_READ */
         gen6_gt_check_fifodbg(dev_priv);
 }
 
+void intel_gt_reset(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (IS_VALLEYVIEW(dev)) {
+                vlv_force_wake_reset(dev_priv);
+        } else if (INTEL_INFO(dev)->gen >= 6) {
+                __gen6_gt_force_wake_reset(dev_priv);
+                if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+                        __gen6_gt_force_wake_mt_reset(dev_priv);
+        }
+}
+
 void intel_gt_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         spin_lock_init(&dev_priv->gt_lock);
 
+        intel_gt_reset(dev);
+
         if (IS_VALLEYVIEW(dev)) {
                 dev_priv->gt.force_wake_get = vlv_force_wake_get;
                 dev_priv->gt.force_wake_put = vlv_force_wake_put;
-        } else if (INTEL_INFO(dev)->gen >= 6) {
+        } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
+                dev_priv->gt.force_wake_get = __gen6_gt_force_wake_mt_get;
+                dev_priv->gt.force_wake_put = __gen6_gt_force_wake_mt_put;
+        } else if (IS_GEN6(dev)) {
                 dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
                 dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
+        }
+        INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
+                          intel_gen6_powersave_work);
+}
 
-                /* IVB configs may use multi-threaded forcewake */
-                if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
-                        u32 ecobus;
-
-                        /* A small trick here - if the bios hasn't configured
-                         * MT forcewake, and if the device is in RC6, then
-                         * force_wake_mt_get will not wake the device and the
-                         * ECOBUS read will return zero. Which will be
-                         * (correctly) interpreted by the test below as MT
-                         * forcewake being disabled.
-                         */
-                        mutex_lock(&dev->struct_mutex);
-                        __gen6_gt_force_wake_mt_get(dev_priv);
-                        ecobus = I915_READ_NOTRACE(ECOBUS);
-                        __gen6_gt_force_wake_mt_put(dev_priv);
-                        mutex_unlock(&dev->struct_mutex);
-
-                        if (ecobus & FORCEWAKE_MT_ENABLE) {
-                                DRM_DEBUG_KMS("Using MT version of forcewake\n");
-                                dev_priv->gt.force_wake_get =
-                                        __gen6_gt_force_wake_mt_get;
-                                dev_priv->gt.force_wake_put =
-                                        __gen6_gt_force_wake_mt_put;
-                        }
-                }
+int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val)
+{
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+
+        if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
+                DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
+                return -EAGAIN;
         }
+
+        I915_WRITE(GEN6_PCODE_DATA, *val);
+        I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
+
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500)) {
+                DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
+                return -ETIMEDOUT;
+        }
+
+        *val = I915_READ(GEN6_PCODE_DATA);
+        I915_WRITE(GEN6_PCODE_DATA, 0);
+
+        return 0;
 }
 
+int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
+{
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+
+        if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
+                DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
+                return -EAGAIN;
+        }
+
+        I915_WRITE(GEN6_PCODE_DATA, val);
+        I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
+
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500)) {
+                DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
+                return -ETIMEDOUT;
+        }
+
+        I915_WRITE(GEN6_PCODE_DATA, 0);
+
+        return 0;
+}