drm/i915: Reduce duplicated forcewake logic

Introduce a structure to track the individual forcewake domains and use
that to eliminate duplicate logic.

v2: - Rebase on latest dinq (Mika)
    - for_each_fw_domain macro (Mika)
    - Handle reset atomically, keeping the timer running (Mika)
    - for_each_fw_domain parameter ordering (Chris)
    - defer timer on new register access (Mika)

v3: - Fix forcewake_reset/get race by waiting pending timers

v4: - cond_resched and verbose warning on timer deletion (Chris)
    - need to run pending timers manually on reset

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v1)
Signed-off-by: Mika Kuoppala <mika.kuoppala@intel.com>
Acked-by: Deepak S <deepak.s@linux.intel.com> (v2)
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

3 files changed, 246 insertions, 318 deletions

diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index ac517eb07c00..4d156e65837f 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -1288,14 +1288,36 @@ static int ironlake_drpc_info(struct seq_file *m)
         return 0;
 }
 
-static int vlv_drpc_info(struct seq_file *m)
+static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
 {
+        struct drm_info_node *node = m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_uncore_forcewake_domain *fw_domain;
+        const char *domain_names[] = {
+                "render",
+                "blitter",
+                "media",
+        };
+        int i;
+
+        spin_lock_irq(&dev_priv->uncore.lock);
+        for_each_fw_domain(fw_domain, dev_priv, i) {
+                seq_printf(m, "%s.wake_count = %u\n",
+                           domain_names[i],
+                           fw_domain->wake_count);
+        }
+        spin_unlock_irq(&dev_priv->uncore.lock);
 
+        return 0;
+}
+
+static int vlv_drpc_info(struct seq_file *m)
+{
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 rpmodectl1, rcctl1, pw_status;
-        unsigned fw_rendercount = 0, fw_mediacount = 0;
 
         intel_runtime_pm_get(dev_priv);
 
@@ -1327,22 +1349,11 @@ static int vlv_drpc_info(struct seq_file *m)
         seq_printf(m, "Media RC6 residency since boot: %u\n",
                    I915_READ(VLV_GT_MEDIA_RC6));
 
-        spin_lock_irq(&dev_priv->uncore.lock);
-        fw_rendercount = dev_priv->uncore.fw_rendercount;
-        fw_mediacount = dev_priv->uncore.fw_mediacount;
-        spin_unlock_irq(&dev_priv->uncore.lock);
-
-        seq_printf(m, "Forcewake Render Count = %u\n", fw_rendercount);
-        seq_printf(m, "Forcewake Media Count = %u\n", fw_mediacount);
-
-
-        return 0;
+        return i915_gen6_forcewake_count_info(m, NULL);
 }
 
-
 static int gen6_drpc_info(struct seq_file *m)
 {
-
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1356,7 +1367,7 @@ static int gen6_drpc_info(struct seq_file *m)
         intel_runtime_pm_get(dev_priv);
 
         spin_lock_irq(&dev_priv->uncore.lock);
-        forcewake_count = dev_priv->uncore.forcewake_count;
+        forcewake_count = dev_priv->uncore.fw_domain[FW_DOMAIN_ID_RENDER].wake_count;
         spin_unlock_irq(&dev_priv->uncore.lock);
 
         if (forcewake_count) {
@@ -1984,30 +1995,6 @@ static int i915_execlists(struct seq_file *m, void *data)
         return 0;
 }
 
-static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
-{
-        struct drm_info_node *node = m->private;
-        struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned forcewake_count = 0, fw_rendercount = 0, fw_mediacount = 0;
-
-        spin_lock_irq(&dev_priv->uncore.lock);
-        if (IS_VALLEYVIEW(dev)) {
-                fw_rendercount = dev_priv->uncore.fw_rendercount;
-                fw_mediacount = dev_priv->uncore.fw_mediacount;
-        } else
-                forcewake_count = dev_priv->uncore.forcewake_count;
-        spin_unlock_irq(&dev_priv->uncore.lock);
-
-        if (IS_VALLEYVIEW(dev)) {
-                seq_printf(m, "fw_rendercount = %u\n", fw_rendercount);
-                seq_printf(m, "fw_mediacount = %u\n", fw_mediacount);
-        } else
-                seq_printf(m, "forcewake count = %u\n", forcewake_count);
-
-        return 0;
-}
-
 static const char *swizzle_string(unsigned swizzle)
 {
         switch (swizzle) {
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 3a32e0d81a9f..ac2082b022f3 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -596,20 +596,45 @@ struct intel_uncore_funcs {
                                 uint64_t val, bool trace);
 };
 
+enum {
+        FW_DOMAIN_ID_RENDER = 0,
+        FW_DOMAIN_ID_BLITTER,
+        FW_DOMAIN_ID_MEDIA,
+
+        FW_DOMAIN_ID_COUNT
+};
+
 struct intel_uncore {
         spinlock_t lock; /** lock is also taken in irq contexts. */
 
         struct intel_uncore_funcs funcs;
 
         unsigned fifo_count;
-        unsigned forcewake_count;
-
-        unsigned fw_rendercount;
-        unsigned fw_mediacount;
-        unsigned fw_blittercount;
-
-        struct timer_list force_wake_timer;
-};
+        unsigned fw_domains;
+
+        struct intel_uncore_forcewake_domain {
+                struct drm_i915_private *i915;
+                int id;
+                unsigned wake_count;
+                struct timer_list timer;
+        } fw_domain[FW_DOMAIN_ID_COUNT];
+#define FORCEWAKE_RENDER        (1 << FW_DOMAIN_ID_RENDER)
+#define FORCEWAKE_BLITTER       (1 << FW_DOMAIN_ID_BLITTER)
+#define FORCEWAKE_MEDIA         (1 << FW_DOMAIN_ID_MEDIA)
+#define FORCEWAKE_ALL           (FORCEWAKE_RENDER | \
+                                 FORCEWAKE_BLITTER | \
+                                 FORCEWAKE_MEDIA)
+};
+
+/* Iterate over initialised fw domains */
+#define for_each_fw_domain_mask(domain__, mask__, dev_priv__, i__) \
+        for ((i__) = 0, (domain__) = &(dev_priv__)->uncore.fw_domain[0]; \
+             (i__) < FW_DOMAIN_ID_COUNT; \
+             (i__)++, (domain__) = &(dev_priv__)->uncore.fw_domain[i__]) \
+                if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__)))
+
+#define for_each_fw_domain(domain__, dev_priv__, i__) \
+        for_each_fw_domain_mask(domain__, FORCEWAKE_ALL, dev_priv__, i__)
 
 #define DEV_INFO_FOR_EACH_FLAG(func, sep) \
         func(is_mobile) sep \
@@ -3167,8 +3192,10 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
  * must be set to prevent GT core from power down and stale values being
  * returned.
  */
-void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine);
-void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine);
+void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
+                            unsigned fw_domains);
+void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
+                            unsigned fw_domains);
 void assert_force_wake_inactive(struct drm_i915_private *dev_priv);
 
 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
@@ -3200,13 +3227,6 @@ void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val);
 int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val);
 int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val);
 
-#define FORCEWAKE_RENDER        (1 << 0)
-#define FORCEWAKE_MEDIA         (1 << 1)
-#define FORCEWAKE_BLITTER       (1 << 2)
-#define FORCEWAKE_ALL           (FORCEWAKE_RENDER | FORCEWAKE_MEDIA | \
-                                        FORCEWAKE_BLITTER)
-
-
 #define I915_READ8(reg)         dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true)
 #define I915_WRITE8(reg, val)   dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true)
 
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 3d1ffacb7612..b35f3a9e6869 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -67,7 +67,7 @@ static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
 }
 
 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
-                                                        int fw_engine)
+                                     int fw_engine)
 {
         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
                             FORCEWAKE_ACK_TIMEOUT_MS))
@@ -93,7 +93,7 @@ static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
 }
 
 static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
-                                                        int fw_engine)
+                                        int fw_engine)
 {
         u32 forcewake_ack;
 
@@ -129,7 +129,7 @@ static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 }
 
 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
-                                                        int fw_engine)
+                                     int fw_engine)
 {
         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
         /* something from same cacheline, but !FORCEWAKE */
@@ -138,7 +138,7 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 }
 
 static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
-                                                        int fw_engine)
+                                        int fw_engine)
 {
         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
@@ -187,7 +187,7 @@ static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 }
 
 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
-                                                int fw_engine)
+                                 int fw_engine)
 {
         /* Check for Render Engine */
         if (FORCEWAKE_RENDER & fw_engine) {
@@ -227,9 +227,8 @@ static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 }
 
 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
-                                        int fw_engine)
+                                 int fw_engine)
 {
-
         /* Check for Render Engine */
         if (FORCEWAKE_RENDER & fw_engine)
                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
@@ -247,37 +246,6 @@ static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
                 gen6_gt_check_fifodbg(dev_priv);
 }
 
-static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
-{
-        if (fw_engine & FORCEWAKE_RENDER &&
-            dev_priv->uncore.fw_rendercount++ != 0)
-                fw_engine &= ~FORCEWAKE_RENDER;
-        if (fw_engine & FORCEWAKE_MEDIA &&
-            dev_priv->uncore.fw_mediacount++ != 0)
-                fw_engine &= ~FORCEWAKE_MEDIA;
-
-        if (fw_engine)
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);
-}
-
-static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
-{
-        if (fw_engine & FORCEWAKE_RENDER) {
-                WARN_ON(!dev_priv->uncore.fw_rendercount);
-                if (--dev_priv->uncore.fw_rendercount != 0)
-                        fw_engine &= ~FORCEWAKE_RENDER;
-        }
-
-        if (fw_engine & FORCEWAKE_MEDIA) {
-                WARN_ON(!dev_priv->uncore.fw_mediacount);
-                if (--dev_priv->uncore.fw_mediacount != 0)
-                        fw_engine &= ~FORCEWAKE_MEDIA;
-        }
-
-        if (fw_engine)
-                dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);
-}
-
 static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
 {
         __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9,
@@ -367,80 +335,72 @@ __gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 }
 
-static void
-gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
+static void gen6_force_wake_timer(unsigned long arg)
 {
-        if (FORCEWAKE_RENDER & fw_engine) {
-                if (dev_priv->uncore.fw_rendercount++ == 0)
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                        FORCEWAKE_RENDER);
-        }
+        struct intel_uncore_forcewake_domain *domain = (void *)arg;
+        unsigned long irqflags;
 
-        if (FORCEWAKE_MEDIA & fw_engine) {
-                if (dev_priv->uncore.fw_mediacount++ == 0)
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                        FORCEWAKE_MEDIA);
-        }
+        assert_device_not_suspended(domain->i915);
 
-        if (FORCEWAKE_BLITTER & fw_engine) {
-                if (dev_priv->uncore.fw_blittercount++ == 0)
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                        FORCEWAKE_BLITTER);
-        }
+        spin_lock_irqsave(&domain->i915->uncore.lock, irqflags);
+        if (WARN_ON(domain->wake_count == 0))
+                domain->wake_count++;
+
+        if (--domain->wake_count == 0)
+                domain->i915->uncore.funcs.force_wake_put(domain->i915,
+                                                          1 << domain->id);
+
+        spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags);
 }
 
-static void
-gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
+void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
 {
-        if (FORCEWAKE_RENDER & fw_engine) {
-                WARN_ON(dev_priv->uncore.fw_rendercount == 0);
-                if (--dev_priv->uncore.fw_rendercount == 0)
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                        FORCEWAKE_RENDER);
-        }
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long irqflags, fw = 0;
+        struct intel_uncore_forcewake_domain *domain;
+        int id, active_domains, retry_count = 100;
 
-        if (FORCEWAKE_MEDIA & fw_engine) {
-                WARN_ON(dev_priv->uncore.fw_mediacount == 0);
-                if (--dev_priv->uncore.fw_mediacount == 0)
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                        FORCEWAKE_MEDIA);
-        }
+        /* Hold uncore.lock across reset to prevent any register access
+         * with forcewake not set correctly. Wait until all pending
+         * timers are run before holding.
+         */
+        while (1) {
+                active_domains = 0;
 
-        if (FORCEWAKE_BLITTER & fw_engine) {
-                WARN_ON(dev_priv->uncore.fw_blittercount == 0);
-                if (--dev_priv->uncore.fw_blittercount == 0)
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                        FORCEWAKE_BLITTER);
-        }
-}
+                for_each_fw_domain(domain, dev_priv, id) {
+                        if (del_timer_sync(&domain->timer) == 0)
+                                continue;
 
-static void gen6_force_wake_timer(unsigned long arg)
-{
-        struct drm_i915_private *dev_priv = (void *)arg;
-        unsigned long irqflags;
+                        gen6_force_wake_timer((unsigned long)domain);
+                }
 
-        assert_device_not_suspended(dev_priv);
+                spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
-        WARN_ON(!dev_priv->uncore.forcewake_count);
+                for_each_fw_domain(domain, dev_priv, id) {
+                        if (timer_pending(&domain->timer))
+                                active_domains |= (1 << id);
+                }
 
-        if (--dev_priv->uncore.forcewake_count == 0)
-                dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
-        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
-}
+                if (active_domains == 0)
+                        break;
 
-void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long irqflags;
+                if (--retry_count == 0) {
+                        DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
+                        break;
+                }
 
-        if (del_timer_sync(&dev_priv->uncore.force_wake_timer))
-                gen6_force_wake_timer((unsigned long)dev_priv);
+                spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+                cond_resched();
+        }
 
-        /* Hold uncore.lock across reset to prevent any register access
-         * with forcewake not set correctly
-         */
-        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+        WARN_ON(active_domains);
+
+        for_each_fw_domain(domain, dev_priv, id)
+                if (domain->wake_count)
+                        fw |= 1 << id;
+
+        if (fw)
+                dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
 
         if (IS_VALLEYVIEW(dev))
                 vlv_force_wake_reset(dev_priv);
@@ -454,28 +414,6 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
                 __gen9_gt_force_wake_mt_reset(dev_priv);
 
         if (restore) { /* If reset with a user forcewake, try to restore */
-                unsigned fw = 0;
-
-                if (IS_VALLEYVIEW(dev)) {
-                        if (dev_priv->uncore.fw_rendercount)
-                                fw |= FORCEWAKE_RENDER;
-
-                        if (dev_priv->uncore.fw_mediacount)
-                                fw |= FORCEWAKE_MEDIA;
-                } else if (IS_GEN9(dev)) {
-                        if (dev_priv->uncore.fw_rendercount)
-                                fw |= FORCEWAKE_RENDER;
-
-                        if (dev_priv->uncore.fw_mediacount)
-                                fw |= FORCEWAKE_MEDIA;
-
-                        if (dev_priv->uncore.fw_blittercount)
-                                fw |= FORCEWAKE_BLITTER;
-                } else {
-                        if (dev_priv->uncore.forcewake_count)
-                                fw = FORCEWAKE_ALL;
-                }
-
                 if (fw)
                         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 
@@ -485,6 +423,9 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
                                 GT_FIFO_FREE_ENTRIES_MASK;
         }
 
+        if (!restore)
+                assert_force_wake_inactive(dev_priv);
+
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
@@ -533,53 +474,59 @@ void intel_uncore_sanitize(struct drm_device *dev)
  * be called at the beginning of the sequence followed by a call to
  * gen6_gt_force_wake_put() at the end of the sequence.
  */
-void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
+void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
+                            unsigned fw_domains)
 {
         unsigned long irqflags;
+        struct intel_uncore_forcewake_domain *domain;
+        int id;
 
         if (!dev_priv->uncore.funcs.force_wake_get)
                 return;
 
         WARN_ON(dev_priv->pm.suspended);
 
+        fw_domains &= dev_priv->uncore.fw_domains;
+
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-        if (IS_GEN9(dev_priv->dev)) {
-                gen9_force_wake_get(dev_priv, fw_engine);
-        } else if (IS_VALLEYVIEW(dev_priv->dev)) {
-                vlv_force_wake_get(dev_priv, fw_engine);
-        } else {
-                if (dev_priv->uncore.forcewake_count++ == 0)
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                              FORCEWAKE_ALL);
+        for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {
+                if (domain->wake_count++)
+                        fw_domains &= ~(1 << id);
         }
 
+        if (fw_domains)
+                dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
 /*
  * see gen6_gt_force_wake_get()
  */
-void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
+void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
+                            unsigned fw_domains)
 {
         unsigned long irqflags;
+        struct intel_uncore_forcewake_domain *domain;
+        int id;
 
         if (!dev_priv->uncore.funcs.force_wake_put)
                 return;
 
+        fw_domains &= dev_priv->uncore.fw_domains;
+
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-        if (IS_GEN9(dev_priv->dev)) {
-                gen9_force_wake_put(dev_priv, fw_engine);
-        } else if (IS_VALLEYVIEW(dev_priv->dev)) {
-                vlv_force_wake_put(dev_priv, fw_engine);
-        } else {
-                WARN_ON(!dev_priv->uncore.forcewake_count);
-                if (--dev_priv->uncore.forcewake_count == 0) {
-                        dev_priv->uncore.forcewake_count++;
-                        mod_timer_pinned(&dev_priv->uncore.force_wake_timer,
-                                         jiffies + 1);
-                }
+        for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {
+                if (WARN_ON(domain->wake_count == 0))
+                        continue;
+
+                if (--domain->wake_count)
+                        continue;
+
+                domain->wake_count++;
+                mod_timer_pinned(&domain->timer, jiffies + 1);
         }
 
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
@@ -587,10 +534,14 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 
 void assert_force_wake_inactive(struct drm_i915_private *dev_priv)
 {
+        struct intel_uncore_forcewake_domain *domain;
+        int i;
+
         if (!dev_priv->uncore.funcs.force_wake_get)
                 return;
 
-        WARN_ON(dev_priv->uncore.forcewake_count > 0);
+        for_each_fw_domain(domain, dev_priv, i)
+                WARN_ON(domain->wake_count);
 }
 
 /* We give fast paths for the really cool registers */
@@ -753,19 +704,37 @@ __gen2_read(64)
         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
         return val
 
+static inline void __force_wake_get(struct drm_i915_private *dev_priv,
+                                    unsigned fw_domains)
+{
+        struct intel_uncore_forcewake_domain *domain;
+        int i;
+
+        if (WARN_ON(!fw_domains))
+                return;
+
+        /* Ideally GCC would be constant-fold and eliminate this loop */
+        for_each_fw_domain_mask(domain, fw_domains, dev_priv, i) {
+                if (domain->wake_count) {
+                        fw_domains &= ~(1 << i);
+                        continue;
+                }
+
+                domain->wake_count++;
+                mod_timer_pinned(&domain->timer, jiffies + 1);
+        }
+
+        if (fw_domains)
+                dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+}
+
 #define __gen6_read(x) \
 static u##x \
 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
         GEN6_READ_HEADER(x); \
         hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
-        if (dev_priv->uncore.forcewake_count == 0 && \
-            NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                                      FORCEWAKE_ALL); \
-                dev_priv->uncore.forcewake_count++; \
-                mod_timer_pinned(&dev_priv->uncore.force_wake_timer, \
-                                 jiffies + 1); \
-        } \
+        if (NEEDS_FORCE_WAKE((dev_priv), (reg))) \
+                __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
         val = __raw_i915_read##x(dev_priv, reg); \
         hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
         GEN6_READ_FOOTER; \
@@ -774,45 +743,27 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 #define __vlv_read(x) \
 static u##x \
 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
-        unsigned fwengine = 0; \
         GEN6_READ_HEADER(x); \
-        if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \
-                if (dev_priv->uncore.fw_rendercount == 0) \
-                        fwengine = FORCEWAKE_RENDER; \
-        } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \
-                if (dev_priv->uncore.fw_mediacount == 0) \
-                        fwengine = FORCEWAKE_MEDIA; \
-        }  \
-        if (fwengine) \
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
+        if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) \
+                __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+        else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) \
+                __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
         val = __raw_i915_read##x(dev_priv, reg); \
-        if (fwengine) \
-                dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
         GEN6_READ_FOOTER; \
 }
 
 #define __chv_read(x) \
 static u##x \
 chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
-        unsigned fwengine = 0; \
         GEN6_READ_HEADER(x); \
-        if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
-                if (dev_priv->uncore.fw_rendercount == 0) \
-                        fwengine = FORCEWAKE_RENDER; \
-        } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
-                if (dev_priv->uncore.fw_mediacount == 0) \
-                        fwengine = FORCEWAKE_MEDIA; \
-        } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
-                if (dev_priv->uncore.fw_rendercount == 0) \
-                        fwengine |= FORCEWAKE_RENDER; \
-                if (dev_priv->uncore.fw_mediacount == 0) \
-                        fwengine |= FORCEWAKE_MEDIA; \
-        } \
-        if (fwengine) \
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
+        if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
+                __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+        else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
+                __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
+        else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
+                __force_wake_get(dev_priv, \
+                                 FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
         val = __raw_i915_read##x(dev_priv, reg); \
-        if (fwengine) \
-                dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
         GEN6_READ_FOOTER; \
 }
 
@@ -822,32 +773,21 @@ chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 #define __gen9_read(x) \
 static u##x \
 gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+        unsigned fw_engine; \
         GEN6_READ_HEADER(x); \
-        if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
-                val = __raw_i915_read##x(dev_priv, reg); \
-        } else { \
-                unsigned fwengine = 0; \
-                if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_rendercount == 0) \
-                                fwengine = FORCEWAKE_RENDER; \
-                } else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_mediacount == 0) \
-                                fwengine = FORCEWAKE_MEDIA; \
-                } else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_rendercount == 0) \
-                                fwengine |= FORCEWAKE_RENDER; \
-                        if (dev_priv->uncore.fw_mediacount == 0) \
-                                fwengine |= FORCEWAKE_MEDIA; \
-                } else { \
-                        if (dev_priv->uncore.fw_blittercount == 0) \
-                                fwengine = FORCEWAKE_BLITTER; \
-                } \
-                if (fwengine) \
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
-                val = __raw_i915_read##x(dev_priv, reg); \
-                if (fwengine) \
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
-        } \
+        if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)))   \
+                fw_engine = 0; \
+        else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg))       \
+                fw_engine = FORCEWAKE_RENDER; \
+        else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
+                fw_engine = FORCEWAKE_MEDIA; \
+        else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
+                fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
+        else \
+                fw_engine = FORCEWAKE_BLITTER; \
+        if (fw_engine) \
+                __force_wake_get(dev_priv, fw_engine); \
+        val = __raw_i915_read##x(dev_priv, reg); \
         GEN6_READ_FOOTER; \
 }
 
@@ -981,17 +921,9 @@ static void \
 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
         GEN6_WRITE_HEADER; \
         hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
-        if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \
-                if (dev_priv->uncore.forcewake_count == 0) \
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                                              FORCEWAKE_ALL); \
-                __raw_i915_write##x(dev_priv, reg, val); \
-                if (dev_priv->uncore.forcewake_count == 0) \
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv, \
-                                                              FORCEWAKE_ALL); \
-        } else { \
-                __raw_i915_write##x(dev_priv, reg, val); \
-        } \
+        if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) \
+                __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+        __raw_i915_write##x(dev_priv, reg, val); \
         hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
         hsw_unclaimed_reg_detect(dev_priv); \
         GEN6_WRITE_FOOTER; \
@@ -1000,28 +932,17 @@ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace
 #define __chv_write(x) \
 static void \
 chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
-        unsigned fwengine = 0; \
         bool shadowed = is_gen8_shadowed(dev_priv, reg); \
         GEN6_WRITE_HEADER; \
         if (!shadowed) { \
-                if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_rendercount == 0) \
-                                fwengine = FORCEWAKE_RENDER; \
-                } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_mediacount == 0) \
-                                fwengine = FORCEWAKE_MEDIA; \
-                } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_rendercount == 0) \
-                                fwengine |= FORCEWAKE_RENDER; \
-                        if (dev_priv->uncore.fw_mediacount == 0) \
-                                fwengine |= FORCEWAKE_MEDIA; \
-                } \
+                if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
+                        __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+                else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
+                        __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
+                else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
+                        __force_wake_get(dev_priv, FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
         } \
-        if (fwengine) \
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
         __raw_i915_write##x(dev_priv, reg, val); \
-        if (fwengine) \
-                dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
         GEN6_WRITE_FOOTER; \
 }
 
@@ -1052,35 +973,22 @@ static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 static void \
 gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \
                 bool trace) { \
+        unsigned fw_engine; \
         GEN6_WRITE_HEADER; \
-        if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \
-                        is_gen9_shadowed(dev_priv, reg)) { \
-                __raw_i915_write##x(dev_priv, reg, val); \
-        } else { \
-                unsigned fwengine = 0; \
-                if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_rendercount == 0) \
-                                fwengine = FORCEWAKE_RENDER; \
-                } else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_mediacount == 0) \
-                                fwengine = FORCEWAKE_MEDIA; \
-                } else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \
-                        if (dev_priv->uncore.fw_rendercount == 0) \
-                                fwengine |= FORCEWAKE_RENDER; \
-                        if (dev_priv->uncore.fw_mediacount == 0) \
-                                fwengine |= FORCEWAKE_MEDIA; \
-                } else { \
-                        if (dev_priv->uncore.fw_blittercount == 0) \
-                                fwengine = FORCEWAKE_BLITTER; \
-                } \
-                if (fwengine) \
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                        fwengine); \
-                __raw_i915_write##x(dev_priv, reg, val); \
-                if (fwengine) \
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv, \
-                                        fwengine); \
-        } \
+        if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \
+            is_gen9_shadowed(dev_priv, reg)) \
+                fw_engine = 0; \
+        else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
+                fw_engine = FORCEWAKE_RENDER; \
+        else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
+                fw_engine = FORCEWAKE_MEDIA; \
+        else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
+                fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
+        else \
+                fw_engine = FORCEWAKE_BLITTER; \
+        if (fw_engine) \
+                __force_wake_get(dev_priv, fw_engine); \
+        __raw_i915_write##x(dev_priv, reg, val); \
         GEN6_WRITE_FOOTER; \
 }
 
@@ -1132,21 +1040,24 @@ do { \
 void intel_uncore_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-
-        setup_timer(&dev_priv->uncore.force_wake_timer,
-                    gen6_force_wake_timer, (unsigned long)dev_priv);
+        struct intel_uncore_forcewake_domain *domain;
+        int i;
 
         __intel_uncore_early_sanitize(dev, false);
 
         if (IS_GEN9(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = __gen9_force_wake_get;
                 dev_priv->uncore.funcs.force_wake_put = __gen9_force_wake_put;
+                dev_priv->uncore.fw_domains = FORCEWAKE_RENDER |
+                        FORCEWAKE_BLITTER | FORCEWAKE_MEDIA;
         } else if (IS_VALLEYVIEW(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
+                dev_priv->uncore.fw_domains = FORCEWAKE_RENDER | FORCEWAKE_MEDIA;
         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
                 dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
+                dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
         } else if (IS_IVYBRIDGE(dev)) {
                 u32 ecobus;
 
@@ -1178,11 +1089,21 @@ void intel_uncore_init(struct drm_device *dev)
                         dev_priv->uncore.funcs.force_wake_put =
                                 __gen6_gt_force_wake_put;
                 }
+                dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
         } else if (IS_GEN6(dev)) {
                 dev_priv->uncore.funcs.force_wake_get =
                         __gen6_gt_force_wake_get;
                 dev_priv->uncore.funcs.force_wake_put =
                         __gen6_gt_force_wake_put;
+                dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
+        }
+
+        for_each_fw_domain(domain, dev_priv, i) {
+                domain->i915 = dev_priv;
+                domain->id = i;
+
+                setup_timer(&domain->timer, gen6_force_wake_timer,
+                            (unsigned long)domain);
         }
 
         switch (INTEL_INFO(dev)->gen) {