drm: Rework vblank-wait handling to allow interrupt reduction.

Previously, drivers supporting vblank interrupt waits would run the interrupt
all the time, or all the time that any 3d client was running, preventing the
CPU from sleeping for long when the system was otherwise idle.  Now, interrupts
are disabled any time that no client is waiting on a vblank event. The new
method uses vblank counters on the chipsets when the interrupts are turned
off, rather than counting interrupts, so that we can continue to present
accurate vblank numbers.

Co-author: Michel Dänzer <michel@tungstengraphics.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>

4 files changed, 214 insertions, 115 deletions

diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
index e1678cae9ccb..6157cd4bb436 100644
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ b/drivers/gpu/drm/radeon/radeon_cp.c
@@ -1287,7 +1287,7 @@ static int radeon_do_resume_cp(struct drm_device * dev)
         radeon_cp_init_ring_buffer(dev, dev_priv);
 
         radeon_do_engine_reset(dev);
-        radeon_enable_interrupt(dev);
+        radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
 
         DRM_DEBUG("radeon_do_resume_cp() complete\n");
 
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 637bd7faf132..71af746a4e47 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -52,6 +52,28 @@ static int dri_library_name(struct drm_device *dev, char *buf)
                         "r300"));
 }
 
+static int radeon_suspend(struct drm_device *dev, pm_message_t state)
+{
+        drm_radeon_private_t *dev_priv = dev->dev_private;
+
+        /* Disable *all* interrupts */
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690)
+                RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
+        RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
+        return 0;
+}
+
+static int radeon_resume(struct drm_device *dev)
+{
+        drm_radeon_private_t *dev_priv = dev->dev_private;
+
+        /* Restore interrupt registers */
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690)
+                RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg);
+        RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg);
+        return 0;
+}
+
 static struct pci_device_id pciidlist[] = {
         radeon_PCI_IDS
 };
@@ -59,8 +81,7 @@ static struct pci_device_id pciidlist[] = {
 static struct drm_driver driver = {
         .driver_features =
             DRIVER_USE_AGP | DRIVER_USE_MTRR | DRIVER_PCI_DMA | DRIVER_SG |
-            DRIVER_HAVE_IRQ | DRIVER_HAVE_DMA | DRIVER_IRQ_SHARED |
-            DRIVER_IRQ_VBL | DRIVER_IRQ_VBL2,
+            DRIVER_HAVE_IRQ | DRIVER_HAVE_DMA | DRIVER_IRQ_SHARED,
         .dev_priv_size = sizeof(drm_radeon_buf_priv_t),
         .load = radeon_driver_load,
         .firstopen = radeon_driver_firstopen,
@@ -69,8 +90,11 @@ static struct drm_driver driver = {
         .postclose = radeon_driver_postclose,
         .lastclose = radeon_driver_lastclose,
         .unload = radeon_driver_unload,
-        .vblank_wait = radeon_driver_vblank_wait,
-        .vblank_wait2 = radeon_driver_vblank_wait2,
+        .suspend = radeon_suspend,
+        .resume = radeon_resume,
+        .get_vblank_counter = radeon_get_vblank_counter,
+        .enable_vblank = radeon_enable_vblank,
+        .disable_vblank = radeon_disable_vblank,
         .dri_library_name = dri_library_name,
         .irq_preinstall = radeon_driver_irq_preinstall,
         .irq_postinstall = radeon_driver_irq_postinstall,
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index 099381693175..d7e9c6cc6a1a 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -378,17 +378,17 @@ extern void radeon_mem_release(struct drm_file *file_priv,
                                struct mem_block *heap);
 
                                 /* radeon_irq.c */
+extern void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state);
 extern int radeon_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv);
 extern int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv);
 
 extern void radeon_do_release(struct drm_device * dev);
-extern int radeon_driver_vblank_wait(struct drm_device * dev,
-                                     unsigned int *sequence);
-extern int radeon_driver_vblank_wait2(struct drm_device * dev,
-                                      unsigned int *sequence);
+extern u32 radeon_get_vblank_counter(struct drm_device *dev, int crtc);
+extern int radeon_enable_vblank(struct drm_device *dev, int crtc);
+extern void radeon_disable_vblank(struct drm_device *dev, int crtc);
 extern irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS);
 extern void radeon_driver_irq_preinstall(struct drm_device * dev);
-extern void radeon_driver_irq_postinstall(struct drm_device * dev);
+extern int radeon_driver_irq_postinstall(struct drm_device *dev);
 extern void radeon_driver_irq_uninstall(struct drm_device * dev);
 extern void radeon_enable_interrupt(struct drm_device *dev);
 extern int radeon_vblank_crtc_get(struct drm_device *dev);
@@ -397,19 +397,22 @@ extern int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value);
 extern int radeon_driver_load(struct drm_device *dev, unsigned long flags);
 extern int radeon_driver_unload(struct drm_device *dev);
 extern int radeon_driver_firstopen(struct drm_device *dev);
-extern void radeon_driver_preclose(struct drm_device * dev, struct drm_file *file_priv);
-extern void radeon_driver_postclose(struct drm_device * dev, struct drm_file * filp);
+extern void radeon_driver_preclose(struct drm_device *dev,
+                                   struct drm_file *file_priv);
+extern void radeon_driver_postclose(struct drm_device *dev,
+                                    struct drm_file *file_priv);
 extern void radeon_driver_lastclose(struct drm_device * dev);
-extern int radeon_driver_open(struct drm_device * dev, struct drm_file * filp_priv);
+extern int radeon_driver_open(struct drm_device *dev,
+                              struct drm_file *file_priv);
 extern long radeon_compat_ioctl(struct file *filp, unsigned int cmd,
                                 unsigned long arg);
 
 /* r300_cmdbuf.c */
 extern void r300_init_reg_flags(struct drm_device *dev);
 
-extern int r300_do_cp_cmdbuf(struct drm_device * dev,
+extern int r300_do_cp_cmdbuf(struct drm_device *dev,
                              struct drm_file *file_priv,
-                             drm_radeon_kcmd_buffer_t * cmdbuf);
+                             drm_radeon_kcmd_buffer_t *cmdbuf);
 
 /* Flags for stats.boxes
  */
@@ -623,6 +626,7 @@ extern int r300_do_cp_cmdbuf(struct drm_device * dev,
 #       define RADEON_SW_INT_TEST               (1 << 25)
 #       define RADEON_SW_INT_TEST_ACK           (1 << 25)
 #       define RADEON_SW_INT_FIRE               (1 << 26)
+#       define R500_DISPLAY_INT_STATUS          (1 << 0)
 
 #define RADEON_HOST_PATH_CNTL           0x0130
 #       define RADEON_HDP_SOFT_RESET            (1 << 26)
@@ -1116,6 +1120,9 @@ extern int r300_do_cp_cmdbuf(struct drm_device * dev,
 
 #define R200_VAP_PVS_CNTL_1               0x22D0
 
+#define RADEON_CRTC_CRNT_FRAME 0x0214
+#define RADEON_CRTC2_CRNT_FRAME 0x0314
+
 #define R500_D1CRTC_STATUS 0x609c
 #define R500_D2CRTC_STATUS 0x689c
 #define R500_CRTC_V_BLANK (1<<0)
diff --git a/drivers/gpu/drm/radeon/radeon_irq.c b/drivers/gpu/drm/radeon/radeon_irq.c
index ee40d197deb7..5079f7054a2f 100644
--- a/drivers/gpu/drm/radeon/radeon_irq.c
+++ b/drivers/gpu/drm/radeon/radeon_irq.c
@@ -27,7 +27,7 @@
  *
  * Authors:
  *    Keith Whitwell <keith@tungstengraphics.com>
- *    Michel Dänzer <michel@daenzer.net>
+ *    Michel D�zer <michel@daenzer.net>
  */
 
 #include "drmP.h"
@@ -35,12 +35,128 @@
 #include "radeon_drm.h"
 #include "radeon_drv.h"
 
-static __inline__ u32 radeon_acknowledge_irqs(drm_radeon_private_t * dev_priv,
-                                              u32 mask)
+void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state)
 {
-        u32 irqs = RADEON_READ(RADEON_GEN_INT_STATUS) & mask;
+        drm_radeon_private_t *dev_priv = dev->dev_private;
+
+        if (state)
+                dev_priv->irq_enable_reg |= mask;
+        else
+                dev_priv->irq_enable_reg &= ~mask;
+
+        RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg);
+}
+
+static void r500_vbl_irq_set_state(struct drm_device *dev, u32 mask, int state)
+{
+        drm_radeon_private_t *dev_priv = dev->dev_private;
+
+        if (state)
+                dev_priv->r500_disp_irq_reg |= mask;
+        else
+                dev_priv->r500_disp_irq_reg &= ~mask;
+
+        RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg);
+}
+
+int radeon_enable_vblank(struct drm_device *dev, int crtc)
+{
+        drm_radeon_private_t *dev_priv = dev->dev_private;
+
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+                switch (crtc) {
+                case 0:
+                        r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 1);
+                        break;
+                case 1:
+                        r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 1);
+                        break;
+                default:
+                        DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+                                  crtc);
+                        return EINVAL;
+                }
+        } else {
+                switch (crtc) {
+                case 0:
+                        radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 1);
+                        break;
+                case 1:
+                        radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 1);
+                        break;
+                default:
+                        DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+                                  crtc);
+                        return EINVAL;
+                }
+        }
+
+        return 0;
+}
+
+void radeon_disable_vblank(struct drm_device *dev, int crtc)
+{
+        drm_radeon_private_t *dev_priv = dev->dev_private;
+
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+                switch (crtc) {
+                case 0:
+                        r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 0);
+                        break;
+                case 1:
+                        r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 0);
+                        break;
+                default:
+                        DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+                                  crtc);
+                        break;
+                }
+        } else {
+                switch (crtc) {
+                case 0:
+                        radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 0);
+                        break;
+                case 1:
+                        radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 0);
+                        break;
+                default:
+                        DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+                                  crtc);
+                        break;
+                }
+        }
+}
+
+static inline u32 radeon_acknowledge_irqs(drm_radeon_private_t *dev_priv, u32 *r500_disp_int)
+{
+        u32 irqs = RADEON_READ(RADEON_GEN_INT_STATUS);
+        u32 irq_mask = RADEON_SW_INT_TEST;
+
+        *r500_disp_int = 0;
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+                /* vbl interrupts in a different place */
+
+                if (irqs & R500_DISPLAY_INT_STATUS) {
+                        /* if a display interrupt */
+                        u32 disp_irq;
+
+                        disp_irq = RADEON_READ(R500_DISP_INTERRUPT_STATUS);
+
+                        *r500_disp_int = disp_irq;
+                        if (disp_irq & R500_D1_VBLANK_INTERRUPT)
+                                RADEON_WRITE(R500_D1MODE_VBLANK_STATUS, R500_VBLANK_ACK);
+                        if (disp_irq & R500_D2_VBLANK_INTERRUPT)
+                                RADEON_WRITE(R500_D2MODE_VBLANK_STATUS, R500_VBLANK_ACK);
+                }
+                irq_mask |= R500_DISPLAY_INT_STATUS;
+        } else
+                irq_mask |= RADEON_CRTC_VBLANK_STAT | RADEON_CRTC2_VBLANK_STAT;
+
+        irqs &= irq_mask;
+
         if (irqs)
                 RADEON_WRITE(RADEON_GEN_INT_STATUS, irqs);
+
         return irqs;
 }
 
@@ -68,44 +184,33 @@ irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS)
         drm_radeon_private_t *dev_priv =
             (drm_radeon_private_t *) dev->dev_private;
         u32 stat;
+        u32 r500_disp_int;
 
         /* Only consider the bits we're interested in - others could be used
          * outside the DRM
          */
-        stat = radeon_acknowledge_irqs(dev_priv, (RADEON_SW_INT_TEST_ACK |
-                                                  RADEON_CRTC_VBLANK_STAT |
-                                                  RADEON_CRTC2_VBLANK_STAT));
+        stat = radeon_acknowledge_irqs(dev_priv, &r500_disp_int);
         if (!stat)
                 return IRQ_NONE;
 
         stat &= dev_priv->irq_enable_reg;
 
         /* SW interrupt */
-        if (stat & RADEON_SW_INT_TEST) {
+        if (stat & RADEON_SW_INT_TEST)
                 DRM_WAKEUP(&dev_priv->swi_queue);
-        }
 
         /* VBLANK interrupt */
-        if (stat & (RADEON_CRTC_VBLANK_STAT|RADEON_CRTC2_VBLANK_STAT)) {
-                int vblank_crtc = dev_priv->vblank_crtc;
-
-                if ((vblank_crtc &
-                     (DRM_RADEON_VBLANK_CRTC1 | DRM_RADEON_VBLANK_CRTC2)) ==
-                    (DRM_RADEON_VBLANK_CRTC1 | DRM_RADEON_VBLANK_CRTC2)) {
-                        if (stat & RADEON_CRTC_VBLANK_STAT)
-                                atomic_inc(&dev->vbl_received);
-                        if (stat & RADEON_CRTC2_VBLANK_STAT)
-                                atomic_inc(&dev->vbl_received2);
-                } else if (((stat & RADEON_CRTC_VBLANK_STAT) &&
-                           (vblank_crtc & DRM_RADEON_VBLANK_CRTC1)) ||
-                           ((stat & RADEON_CRTC2_VBLANK_STAT) &&
-                            (vblank_crtc & DRM_RADEON_VBLANK_CRTC2)))
-                        atomic_inc(&dev->vbl_received);
-
-                DRM_WAKEUP(&dev->vbl_queue);
-                drm_vbl_send_signals(dev);
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+                if (r500_disp_int & R500_D1_VBLANK_INTERRUPT)
+                        drm_handle_vblank(dev, 0);
+                if (r500_disp_int & R500_D2_VBLANK_INTERRUPT)
+                        drm_handle_vblank(dev, 1);
+        } else {
+                if (stat & RADEON_CRTC_VBLANK_STAT)
+                        drm_handle_vblank(dev, 0);
+                if (stat & RADEON_CRTC2_VBLANK_STAT)
+                        drm_handle_vblank(dev, 1);
         }
-
         return IRQ_HANDLED;
 }
 
@@ -144,54 +249,31 @@ static int radeon_wait_irq(struct drm_device * dev, int swi_nr)
         return ret;
 }
 
-static int radeon_driver_vblank_do_wait(struct drm_device * dev,
-                                        unsigned int *sequence, int crtc)
+u32 radeon_get_vblank_counter(struct drm_device *dev, int crtc)
 {
-        drm_radeon_private_t *dev_priv =
-            (drm_radeon_private_t *) dev->dev_private;
-        unsigned int cur_vblank;
-        int ret = 0;
-        int ack = 0;
-        atomic_t *counter;
+        drm_radeon_private_t *dev_priv = dev->dev_private;
+
         if (!dev_priv) {
                 DRM_ERROR("called with no initialization\n");
                 return -EINVAL;
         }
 
-        if (crtc == DRM_RADEON_VBLANK_CRTC1) {
-                counter = &dev->vbl_received;
-                ack |= RADEON_CRTC_VBLANK_STAT;
-        } else if (crtc == DRM_RADEON_VBLANK_CRTC2) {
-                counter = &dev->vbl_received2;
-                ack |= RADEON_CRTC2_VBLANK_STAT;
-        } else
+        if (crtc < 0 || crtc > 1) {
+                DRM_ERROR("Invalid crtc %d\n", crtc);
                 return -EINVAL;
+        }
 
-        radeon_acknowledge_irqs(dev_priv, ack);
-
-        dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-        /* Assume that the user has missed the current sequence number
-         * by about a day rather than she wants to wait for years
-         * using vertical blanks...
-         */
-        DRM_WAIT_ON(ret, dev->vbl_queue, 3 * DRM_HZ,
-                    (((cur_vblank = atomic_read(counter))
-                      - *sequence) <= (1 << 23)));
-
-        *sequence = cur_vblank;
-
-        return ret;
-}
-
-int radeon_driver_vblank_wait(struct drm_device *dev, unsigned int *sequence)
-{
-        return radeon_driver_vblank_do_wait(dev, sequence, DRM_RADEON_VBLANK_CRTC1);
-}
-
-int radeon_driver_vblank_wait2(struct drm_device *dev, unsigned int *sequence)
-{
-        return radeon_driver_vblank_do_wait(dev, sequence, DRM_RADEON_VBLANK_CRTC2);
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+                if (crtc == 0)
+                        return RADEON_READ(R500_D1CRTC_FRAME_COUNT);
+                else
+                        return RADEON_READ(R500_D2CRTC_FRAME_COUNT);
+        } else {
+                if (crtc == 0)
+                        return RADEON_READ(RADEON_CRTC_CRNT_FRAME);
+                else
+                        return RADEON_READ(RADEON_CRTC2_CRNT_FRAME);
+        }
 }
 
 /* Needs the lock as it touches the ring.
@@ -234,46 +316,41 @@ int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_pr
         return radeon_wait_irq(dev, irqwait->irq_seq);
 }
 
-void radeon_enable_interrupt(struct drm_device *dev)
-{
-        drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private;
-
-        dev_priv->irq_enable_reg = RADEON_SW_INT_ENABLE;
-        if (dev_priv->vblank_crtc & DRM_RADEON_VBLANK_CRTC1)
-                dev_priv->irq_enable_reg |= RADEON_CRTC_VBLANK_MASK;
-
-        if (dev_priv->vblank_crtc & DRM_RADEON_VBLANK_CRTC2)
-                dev_priv->irq_enable_reg |= RADEON_CRTC2_VBLANK_MASK;
-
-        RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg);
-        dev_priv->irq_enabled = 1;
-}
-
 /* drm_dma.h hooks
 */
 void radeon_driver_irq_preinstall(struct drm_device * dev)
 {
         drm_radeon_private_t *dev_priv =
             (drm_radeon_private_t *) dev->dev_private;
+        u32 dummy;
 
         /* Disable *all* interrupts */
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690)
+                RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
         RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
 
         /* Clear bits if they're already high */
-        radeon_acknowledge_irqs(dev_priv, (RADEON_SW_INT_TEST_ACK |
-                                           RADEON_CRTC_VBLANK_STAT |
-                                           RADEON_CRTC2_VBLANK_STAT));
+        radeon_acknowledge_irqs(dev_priv, &dummy);
 }
 
-void radeon_driver_irq_postinstall(struct drm_device * dev)
+int radeon_driver_irq_postinstall(struct drm_device *dev)
 {
         drm_radeon_private_t *dev_priv =
             (drm_radeon_private_t *) dev->dev_private;
+        int ret;
 
         atomic_set(&dev_priv->swi_emitted, 0);
         DRM_INIT_WAITQUEUE(&dev_priv->swi_queue);
 
-        radeon_enable_interrupt(dev);
+        ret = drm_vblank_init(dev, 2);
+        if (ret)
+                return ret;
+
+        dev->max_vblank_count = 0x001fffff;
+
+        radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
+
+        return 0;
 }
 
 void radeon_driver_irq_uninstall(struct drm_device * dev)
@@ -285,6 +362,8 @@ void radeon_driver_irq_uninstall(struct drm_device * dev)
 
         dev_priv->irq_enabled = 0;
 
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690)
+                RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
         /* Disable *all* interrupts */
         RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
 }
@@ -293,18 +372,8 @@ void radeon_driver_irq_uninstall(struct drm_device * dev)
 int radeon_vblank_crtc_get(struct drm_device *dev)
 {
         drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private;
-        u32 flag;
-        u32 value;
-
-        flag = RADEON_READ(RADEON_GEN_INT_CNTL);
-        value = 0;
-
-        if (flag & RADEON_CRTC_VBLANK_MASK)
-                value |= DRM_RADEON_VBLANK_CRTC1;
 
-        if (flag & RADEON_CRTC2_VBLANK_MASK)
-                value |= DRM_RADEON_VBLANK_CRTC2;
-        return value;
+        return dev_priv->vblank_crtc;
 }
 
 int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value)
@@ -315,6 +384,5 @@ int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value)
                 return -EINVAL;
         }
         dev_priv->vblank_crtc = (unsigned int)value;
-        radeon_enable_interrupt(dev);
         return 0;
 }