drm/i915: Mask USER interrupts on gen6 (until required)

Otherwise we may consume 20% of the CPU just handling IRQs whilst
rendering. Ouch.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

5 files changed, 113 insertions, 62 deletions

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 972e08e4e054..1f77d8c6c6a2 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1007,12 +1007,6 @@ extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc);
 extern u32 gm45_get_vblank_counter(struct drm_device *dev, int crtc);
 extern int i915_vblank_swap(struct drm_device *dev, void *data,
                             struct drm_file *file_priv);
-extern void i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask);
-extern void i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask);
-extern void ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv,
-                u32 mask);
-extern void ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv,
-                u32 mask);
 
 void
 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 0dadc025b77b..826873a23db0 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -64,26 +64,6 @@
 #define DRM_I915_VBLANK_PIPE_ALL        (DRM_I915_VBLANK_PIPE_A | \
                                          DRM_I915_VBLANK_PIPE_B)
 
-void
-ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->gt_irq_mask & mask) != 0) {
-                dev_priv->gt_irq_mask &= ~mask;
-                I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
-                POSTING_READ(GTIMR);
-        }
-}
-
-void
-ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->gt_irq_mask & mask) != mask) {
-                dev_priv->gt_irq_mask |= mask;
-                I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
-                POSTING_READ(GTIMR);
-        }
-}
-
 /* For display hotplug interrupt */
 static void
 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
@@ -105,26 +85,6 @@ ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
         }
 }
 
-void
-i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->irq_mask & mask) != 0) {
-                dev_priv->irq_mask &= ~mask;
-                I915_WRITE(IMR, dev_priv->irq_mask);
-                POSTING_READ(IMR);
-        }
-}
-
-void
-i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->irq_mask & mask) != mask) {
-                dev_priv->irq_mask |= mask;
-                I915_WRITE(IMR, dev_priv->irq_mask);
-                POSTING_READ(IMR);
-        }
-}
-
 static inline u32
 i915_pipestat(int pipe)
 {
@@ -1673,11 +1633,6 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 
         I915_WRITE(GTIIR, I915_READ(GTIIR));
         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
-        if (IS_GEN6(dev)) {
-                I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_USER_INTERRUPT);
-                I915_WRITE(GEN6_BSD_IMR, ~GEN6_BSD_USER_INTERRUPT);
-                I915_WRITE(GEN6_BLITTER_IMR, ~GEN6_BLITTER_USER_INTERRUPT);
-        }
 
         if (IS_GEN6(dev))
                 render_irqs =
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index ecfb0023f60d..b0ab4247ce48 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -288,6 +288,7 @@
 #define RING_HWS_PGA_GEN6(base) ((base)+0x2080)
 #define RING_ACTHD(base)        ((base)+0x74)
 #define RING_NOPID(base)        ((base)+0x94)
+#define RING_IMR(base)          ((base)+0xa8)
 #define   TAIL_ADDR             0x001FFFF8
 #define   HEAD_WRAP_COUNT       0xFFE00000
 #define   HEAD_WRAP_ONE         0x00200000
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index aa8f6abf16f2..3bff7fb72341 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -485,6 +485,38 @@ pc_render_get_seqno(struct intel_ring_buffer *ring)
         return pc->cpu_page[0];
 }
 
+static void
+ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->gt_irq_mask &= ~mask;
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+        POSTING_READ(GTIMR);
+}
+
+static void
+ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->gt_irq_mask |= mask;
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+        POSTING_READ(GTIMR);
+}
+
+static void
+i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->irq_mask &= ~mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
+        POSTING_READ(IMR);
+}
+
+static void
+i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->irq_mask |= mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
+        POSTING_READ(IMR);
+}
+
 static bool
 render_ring_get_irq(struct intel_ring_buffer *ring)
 {
@@ -499,8 +531,8 @@ render_ring_get_irq(struct intel_ring_buffer *ring)
 
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 if (HAS_PCH_SPLIT(dev))
-                        ironlake_enable_graphics_irq(dev_priv,
-                                                     GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
+                        ironlake_enable_irq(dev_priv,
+                                            GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
                 else
                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
@@ -520,9 +552,9 @@ render_ring_put_irq(struct intel_ring_buffer *ring)
 
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 if (HAS_PCH_SPLIT(dev))
-                        ironlake_disable_graphics_irq(dev_priv,
-                                                      GT_USER_INTERRUPT |
-                                                      GT_PIPE_NOTIFY);
+                        ironlake_disable_irq(dev_priv,
+                                             GT_USER_INTERRUPT |
+                                             GT_PIPE_NOTIFY);
                 else
                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
@@ -596,7 +628,7 @@ ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
                 unsigned long irqflags;
 
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-                ironlake_enable_graphics_irq(dev_priv, flag);
+                ironlake_enable_irq(dev_priv, flag);
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
         }
 
@@ -613,7 +645,46 @@ ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
                 unsigned long irqflags;
 
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-                ironlake_disable_graphics_irq(dev_priv, flag);
+                ironlake_disable_irq(dev_priv, flag);
+                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+        }
+}
+
+static bool
+gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
+{
+        struct drm_device *dev = ring->dev;
+
+        if (!dev->irq_enabled)
+               return false;
+
+        if (atomic_inc_return(&ring->irq_refcount) == 1) {
+                drm_i915_private_t *dev_priv = dev->dev_private;
+                unsigned long irqflags;
+
+                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+                ring->irq_mask &= ~rflag;
+                I915_WRITE_IMR(ring, ring->irq_mask);
+                ironlake_enable_irq(dev_priv, gflag);
+                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+        }
+
+        return true;
+}
+
+static void
+gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
+{
+        struct drm_device *dev = ring->dev;
+
+        if (atomic_dec_and_test(&ring->irq_refcount)) {
+                drm_i915_private_t *dev_priv = dev->dev_private;
+                unsigned long irqflags;
+
+                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+                ring->irq_mask |= rflag;
+                I915_WRITE_IMR(ring, ring->irq_mask);
+                ironlake_disable_irq(dev_priv, gflag);
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
         }
 }
@@ -757,6 +828,7 @@ int intel_init_ring_buffer(struct drm_device *dev,
         INIT_LIST_HEAD(&ring->active_list);
         INIT_LIST_HEAD(&ring->request_list);
         INIT_LIST_HEAD(&ring->gpu_write_list);
+        ring->irq_mask = ~0;
 
         if (I915_NEED_GFX_HWS(dev)) {
                 ret = init_status_page(ring);
@@ -1030,15 +1102,35 @@ gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 }
 
 static bool
+gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_get_irq(ring,
+                                 GT_USER_INTERRUPT,
+                                 GEN6_RENDER_USER_INTERRUPT);
+}
+
+static void
+gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_put_irq(ring,
+                                 GT_USER_INTERRUPT,
+                                 GEN6_RENDER_USER_INTERRUPT);
+}
+
+static bool
 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
 {
-        return ring_get_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
+        return gen6_ring_get_irq(ring,
+                                 GT_GEN6_BSD_USER_INTERRUPT,
+                                 GEN6_BSD_USER_INTERRUPT);
 }
 
 static void
 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
 {
-        ring_put_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
+        return gen6_ring_put_irq(ring,
+                                 GT_GEN6_BSD_USER_INTERRUPT,
+                                 GEN6_BSD_USER_INTERRUPT);
 }
 
 /* ring buffer for Video Codec for Gen6+ */
@@ -1062,13 +1154,17 @@ static const struct intel_ring_buffer gen6_bsd_ring = {
 static bool
 blt_ring_get_irq(struct intel_ring_buffer *ring)
 {
-        return ring_get_irq(ring, GT_BLT_USER_INTERRUPT);
+        return gen6_ring_get_irq(ring,
+                                 GT_BLT_USER_INTERRUPT,
+                                 GEN6_BLITTER_USER_INTERRUPT);
 }
 
 static void
 blt_ring_put_irq(struct intel_ring_buffer *ring)
 {
-        ring_put_irq(ring, GT_BLT_USER_INTERRUPT);
+        gen6_ring_put_irq(ring,
+                          GT_BLT_USER_INTERRUPT,
+                          GEN6_BLITTER_USER_INTERRUPT);
 }
 
 
@@ -1192,6 +1288,8 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
         *ring = render_ring;
         if (INTEL_INFO(dev)->gen >= 6) {
                 ring->add_request = gen6_add_request;
+                ring->irq_get = gen6_render_ring_get_irq;
+                ring->irq_put = gen6_render_ring_put_irq;
         } else if (IS_GEN5(dev)) {
                 ring->add_request = pc_render_add_request;
                 ring->get_seqno = pc_render_get_seqno;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 5969c2ed1028..634f6f84cb57 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -28,6 +28,8 @@ struct  intel_hw_status_page {
 #define I915_READ_CTL(ring) I915_RING_READ(RING_CTL(ring->mmio_base))
 #define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL(ring->mmio_base), val)
 
+#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR(ring->mmio_base), val)
+
 #define I915_READ_NOPID(ring) I915_RING_READ(RING_NOPID(ring->mmio_base))
 #define I915_READ_SYNC_0(ring) I915_RING_READ(RING_SYNC_0(ring->mmio_base))
 #define I915_READ_SYNC_1(ring) I915_RING_READ(RING_SYNC_1(ring->mmio_base))
@@ -52,6 +54,7 @@ struct  intel_ring_buffer {
         int             effective_size;
         struct intel_hw_status_page status_page;
 
+        u32             irq_mask;
         u32             irq_seqno;              /* last seq seem at irq time */
         u32             waiting_seqno;
         u32             sync_seqno[I915_NUM_RINGS-1];