drm/radeon/kms: Fix up vertical blank interrupt support.

Fixes 3D apps timing out in the WAIT_VBLANK ioctl.

AVIVO bits compile-tested only.

Signed-off-by: Michel Dänzer <daenzer@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

9 files changed, 217 insertions, 77 deletions

diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index f1ba8ff41130..e1a6e82b9960 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -254,6 +254,72 @@ void r100_mc_fini(struct radeon_device *rdev)
 
 
 /*
+ * Interrupts
+ */
+int r100_irq_set(struct radeon_device *rdev)
+{
+        uint32_t tmp = 0;
+
+        if (rdev->irq.sw_int) {
+                tmp |= RADEON_SW_INT_ENABLE;
+        }
+        if (rdev->irq.crtc_vblank_int[0]) {
+                tmp |= RADEON_CRTC_VBLANK_MASK;
+        }
+        if (rdev->irq.crtc_vblank_int[1]) {
+                tmp |= RADEON_CRTC2_VBLANK_MASK;
+        }
+        WREG32(RADEON_GEN_INT_CNTL, tmp);
+        return 0;
+}
+
+static inline uint32_t r100_irq_ack(struct radeon_device *rdev)
+{
+        uint32_t irqs = RREG32(RADEON_GEN_INT_STATUS);
+        uint32_t irq_mask = RADEON_SW_INT_TEST | RADEON_CRTC_VBLANK_STAT |
+                RADEON_CRTC2_VBLANK_STAT;
+
+        if (irqs) {
+                WREG32(RADEON_GEN_INT_STATUS, irqs);
+        }
+        return irqs & irq_mask;
+}
+
+int r100_irq_process(struct radeon_device *rdev)
+{
+        uint32_t status;
+
+        status = r100_irq_ack(rdev);
+        if (!status) {
+                return IRQ_NONE;
+        }
+        while (status) {
+                /* SW interrupt */
+                if (status & RADEON_SW_INT_TEST) {
+                        radeon_fence_process(rdev);
+                }
+                /* Vertical blank interrupts */
+                if (status & RADEON_CRTC_VBLANK_STAT) {
+                        drm_handle_vblank(rdev->ddev, 0);
+                }
+                if (status & RADEON_CRTC2_VBLANK_STAT) {
+                        drm_handle_vblank(rdev->ddev, 1);
+                }
+                status = r100_irq_ack(rdev);
+        }
+        return IRQ_HANDLED;
+}
+
+u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc)
+{
+        if (crtc == 0)
+                return RREG32(RADEON_CRTC_CRNT_FRAME);
+        else
+                return RREG32(RADEON_CRTC2_CRNT_FRAME);
+}
+
+
+/*
  * Fence emission
  */
 void r100_fence_ring_emit(struct radeon_device *rdev,
diff --git a/drivers/gpu/drm/radeon/r500_reg.h b/drivers/gpu/drm/radeon/r500_reg.h
index 036691b38cb7..e1d5e0331e19 100644
--- a/drivers/gpu/drm/radeon/r500_reg.h
+++ b/drivers/gpu/drm/radeon/r500_reg.h
@@ -350,6 +350,7 @@
 #define AVIVO_D1CRTC_BLANK_CONTROL                              0x6084
 #define AVIVO_D1CRTC_INTERLACE_CONTROL                          0x6088
 #define AVIVO_D1CRTC_INTERLACE_STATUS                           0x608c
+#define AVIVO_D1CRTC_FRAME_COUNT                                0x60a4
 #define AVIVO_D1CRTC_STEREO_CONTROL                             0x60c4
 
 /* master controls */
@@ -438,14 +439,15 @@
 #       define AVIVO_DC_LB_DISP1_END_ADR_SHIFT  4
 #       define AVIVO_DC_LB_DISP1_END_ADR_MASK   0x7ff
 
-#define R500_DxMODE_INT_MASK 0x6540
-#define R500_D1MODE_INT_MASK (1<<0)
-#define R500_D2MODE_INT_MASK (1<<8)
-
 #define AVIVO_D1MODE_DATA_FORMAT                0x6528
 #       define AVIVO_D1MODE_INTERLEAVE_EN       (1 << 0)
 #define AVIVO_D1MODE_DESKTOP_HEIGHT             0x652C
+#define AVIVO_D1MODE_VBLANK_STATUS              0x6534
+#       define AVIVO_VBLANK_ACK                 (1 << 4)
 #define AVIVO_D1MODE_VLINE_START_END            0x6538
+#define AVIVO_DxMODE_INT_MASK                   0x6540
+#       define AVIVO_D1MODE_INT_MASK            (1 << 0)
+#       define AVIVO_D2MODE_INT_MASK            (1 << 8)
 #define AVIVO_D1MODE_VIEWPORT_START             0x6580
 #define AVIVO_D1MODE_VIEWPORT_SIZE              0x6584
 #define AVIVO_D1MODE_EXT_OVERSCAN_LEFT_RIGHT    0x6588
@@ -475,6 +477,7 @@
 #define AVIVO_D2CRTC_BLANK_CONTROL                              0x6884
 #define AVIVO_D2CRTC_INTERLACE_CONTROL                          0x6888
 #define AVIVO_D2CRTC_INTERLACE_STATUS                           0x688c
+#define AVIVO_D2CRTC_FRAME_COUNT                                0x68a4
 #define AVIVO_D2CRTC_STEREO_CONTROL                             0x68c4
 
 #define AVIVO_D2GRPH_ENABLE                                     0x6900
@@ -497,6 +500,7 @@
 #define AVIVO_D2CUR_SIZE                        0x6c10
 #define AVIVO_D2CUR_POSITION                    0x6c14
 
+#define AVIVO_D2MODE_VBLANK_STATUS              0x6d34
 #define AVIVO_D2MODE_VLINE_START_END            0x6d38
 #define AVIVO_D2MODE_VIEWPORT_START             0x6d80
 #define AVIVO_D2MODE_VIEWPORT_SIZE              0x6d84
@@ -748,4 +752,8 @@
 #       define AVIVO_I2C_EN                                                     (1 << 0)
 #       define AVIVO_I2C_RESET                                          (1 << 8)
 
+#define AVIVO_DISP_INTERRUPT_STATUS                             0x7edc
+#       define AVIVO_D1_VBLANK_INTERRUPT                        (1 << 4)
+#       define AVIVO_D2_VBLANK_INTERRUPT                        (1 << 5)
+
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index b1d945b8ed6c..346112740846 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -574,6 +574,7 @@ struct radeon_asic {
         void (*ring_start)(struct radeon_device *rdev);
         int (*irq_set)(struct radeon_device *rdev);
         int (*irq_process)(struct radeon_device *rdev);
+        u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc);
         void (*fence_ring_emit)(struct radeon_device *rdev, struct radeon_fence *fence);
         int (*cs_parse)(struct radeon_cs_parser *p);
         int (*copy_blit)(struct radeon_device *rdev,
@@ -862,6 +863,7 @@ static inline void radeon_ring_write(struct radeon_device *rdev, uint32_t v)
 #define radeon_ring_start(rdev) (rdev)->asic->ring_start((rdev))
 #define radeon_irq_set(rdev) (rdev)->asic->irq_set((rdev))
 #define radeon_irq_process(rdev) (rdev)->asic->irq_process((rdev))
+#define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->get_vblank_counter((rdev), (crtc))
 #define radeon_fence_ring_emit(rdev, fence) (rdev)->asic->fence_ring_emit((rdev), (fence))
 #define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy_blit((rdev), (s), (d), (np), (f))
 #define radeon_copy_dma(rdev, s, d, np, f) (rdev)->asic->copy_dma((rdev), (s), (d), (np), (f))
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index c0ae2d923254..7ca6c13569b5 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -49,6 +49,7 @@ void r100_vram_info(struct radeon_device *rdev);
 int r100_gpu_reset(struct radeon_device *rdev);
 int r100_mc_init(struct radeon_device *rdev);
 void r100_mc_fini(struct radeon_device *rdev);
+u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc);
 int r100_wb_init(struct radeon_device *rdev);
 void r100_wb_fini(struct radeon_device *rdev);
 int r100_gart_enable(struct radeon_device *rdev);
@@ -96,6 +97,7 @@ static struct radeon_asic r100_asic = {
         .ring_start = &r100_ring_start,
         .irq_set = &r100_irq_set,
         .irq_process = &r100_irq_process,
+        .get_vblank_counter = &r100_get_vblank_counter,
         .fence_ring_emit = &r100_fence_ring_emit,
         .cs_parse = &r100_cs_parse,
         .copy_blit = &r100_copy_blit,
@@ -156,6 +158,7 @@ static struct radeon_asic r300_asic = {
         .ring_start = &r300_ring_start,
         .irq_set = &r100_irq_set,
         .irq_process = &r100_irq_process,
+        .get_vblank_counter = &r100_get_vblank_counter,
         .fence_ring_emit = &r300_fence_ring_emit,
         .cs_parse = &r300_cs_parse,
         .copy_blit = &r100_copy_blit,
@@ -196,6 +199,7 @@ static struct radeon_asic r420_asic = {
         .ring_start = &r300_ring_start,
         .irq_set = &r100_irq_set,
         .irq_process = &r100_irq_process,
+        .get_vblank_counter = &r100_get_vblank_counter,
         .fence_ring_emit = &r300_fence_ring_emit,
         .cs_parse = &r300_cs_parse,
         .copy_blit = &r100_copy_blit,
@@ -243,6 +247,7 @@ static struct radeon_asic rs400_asic = {
         .ring_start = &r300_ring_start,
         .irq_set = &r100_irq_set,
         .irq_process = &r100_irq_process,
+        .get_vblank_counter = &r100_get_vblank_counter,
         .fence_ring_emit = &r300_fence_ring_emit,
         .cs_parse = &r300_cs_parse,
         .copy_blit = &r100_copy_blit,
@@ -266,6 +271,8 @@ void rs600_vram_info(struct radeon_device *rdev);
 int rs600_mc_init(struct radeon_device *rdev);
 void rs600_mc_fini(struct radeon_device *rdev);
 int rs600_irq_set(struct radeon_device *rdev);
+int rs600_irq_process(struct radeon_device *rdev);
+u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc);
 int rs600_gart_enable(struct radeon_device *rdev);
 void rs600_gart_disable(struct radeon_device *rdev);
 void rs600_gart_tlb_flush(struct radeon_device *rdev);
@@ -291,7 +298,8 @@ static struct radeon_asic rs600_asic = {
         .cp_disable = &r100_cp_disable,
         .ring_start = &r300_ring_start,
         .irq_set = &rs600_irq_set,
-        .irq_process = &r100_irq_process,
+        .irq_process = &rs600_irq_process,
+        .get_vblank_counter = &rs600_get_vblank_counter,
         .fence_ring_emit = &r300_fence_ring_emit,
         .cs_parse = &r300_cs_parse,
         .copy_blit = &r100_copy_blit,
@@ -334,7 +342,8 @@ static struct radeon_asic rs690_asic = {
         .cp_disable = &r100_cp_disable,
         .ring_start = &r300_ring_start,
         .irq_set = &rs600_irq_set,
-        .irq_process = &r100_irq_process,
+        .irq_process = &rs600_irq_process,
+        .get_vblank_counter = &rs600_get_vblank_counter,
         .fence_ring_emit = &r300_fence_ring_emit,
         .cs_parse = &r300_cs_parse,
         .copy_blit = &r100_copy_blit,
@@ -382,8 +391,9 @@ static struct radeon_asic rv515_asic = {
         .cp_fini = &r100_cp_fini,
         .cp_disable = &r100_cp_disable,
         .ring_start = &rv515_ring_start,
-        .irq_set = &r100_irq_set,
-        .irq_process = &r100_irq_process,
+        .irq_set = &rs600_irq_set,
+        .irq_process = &rs600_irq_process,
+        .get_vblank_counter = &rs600_get_vblank_counter,
         .fence_ring_emit = &r300_fence_ring_emit,
         .cs_parse = &r300_cs_parse,
         .copy_blit = &r100_copy_blit,
@@ -424,8 +434,9 @@ static struct radeon_asic r520_asic = {
         .cp_fini = &r100_cp_fini,
         .cp_disable = &r100_cp_disable,
         .ring_start = &rv515_ring_start,
-        .irq_set = &r100_irq_set,
-        .irq_process = &r100_irq_process,
+        .irq_set = &rs600_irq_set,
+        .irq_process = &rs600_irq_process,
+        .get_vblank_counter = &rs600_get_vblank_counter,
         .fence_ring_emit = &r300_fence_ring_emit,
         .cs_parse = &r300_cs_parse,
         .copy_blit = &r100_copy_blit,
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 491d569deb0e..9805e4b6ca1b 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -32,60 +32,6 @@
 #include "radeon.h"
 #include "atom.h"
 
-static inline uint32_t r100_irq_ack(struct radeon_device *rdev)
-{
-        uint32_t irqs = RREG32(RADEON_GEN_INT_STATUS);
-        uint32_t irq_mask = RADEON_SW_INT_TEST;
-
-        if (irqs) {
-                WREG32(RADEON_GEN_INT_STATUS, irqs);
-        }
-        return irqs & irq_mask;
-}
-
-int r100_irq_set(struct radeon_device *rdev)
-{
-        uint32_t tmp = 0;
-
-        if (rdev->irq.sw_int) {
-                tmp |= RADEON_SW_INT_ENABLE;
-        }
-        /* Todo go through CRTC and enable vblank int or not */
-        WREG32(RADEON_GEN_INT_CNTL, tmp);
-        return 0;
-}
-
-int r100_irq_process(struct radeon_device *rdev)
-{
-        uint32_t status;
-
-        status = r100_irq_ack(rdev);
-        if (!status) {
-                return IRQ_NONE;
-        }
-        while (status) {
-                /* SW interrupt */
-                if (status & RADEON_SW_INT_TEST) {
-                        radeon_fence_process(rdev);
-                }
-                status = r100_irq_ack(rdev);
-        }
-        return IRQ_HANDLED;
-}
-
-int rs600_irq_set(struct radeon_device *rdev)
-{
-        uint32_t tmp = 0;
-
-        if (rdev->irq.sw_int) {
-                tmp |= RADEON_SW_INT_ENABLE;
-        }
-        WREG32(RADEON_GEN_INT_CNTL, tmp);
-        /* Todo go through CRTC and enable vblank int or not */
-        WREG32(R500_DxMODE_INT_MASK, 0);
-        return 0;
-}
-
 irqreturn_t radeon_driver_irq_handler_kms(DRM_IRQ_ARGS)
 {
         struct drm_device *dev = (struct drm_device *) arg;
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 3357110e30ce..d2764bf6b2a2 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -141,19 +141,42 @@ void radeon_driver_preclose_kms(struct drm_device *dev,
  */
 u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc)
 {
-        /* FIXME: implement */
-        return 0;
+        struct radeon_device *rdev = dev->dev_private;
+
+        if (crtc < 0 || crtc > 1) {
+                DRM_ERROR("Invalid crtc %d\n", crtc);
+                return -EINVAL;
+        }
+
+        return radeon_get_vblank_counter(rdev, crtc);
 }
 
 int radeon_enable_vblank_kms(struct drm_device *dev, int crtc)
 {
-        /* FIXME: implement */
-        return 0;
+        struct radeon_device *rdev = dev->dev_private;
+
+        if (crtc < 0 || crtc > 1) {
+                DRM_ERROR("Invalid crtc %d\n", crtc);
+                return -EINVAL;
+        }
+
+        rdev->irq.crtc_vblank_int[crtc] = true;
+
+        return radeon_irq_set(rdev);
 }
 
 void radeon_disable_vblank_kms(struct drm_device *dev, int crtc)
 {
-        /* FIXME: implement */
+        struct radeon_device *rdev = dev->dev_private;
+
+        if (crtc < 0 || crtc > 1) {
+                DRM_ERROR("Invalid crtc %d\n", crtc);
+                return;
+        }
+
+        rdev->irq.crtc_vblank_int[crtc] = false;
+
+        radeon_irq_set(rdev);
 }
 
 
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 7d06dc98a42a..0da72f18fd3a 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -310,10 +310,13 @@ void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
                                                                          RADEON_CRTC_DISP_REQ_EN_B));
                         WREG32_P(RADEON_CRTC_EXT_CNTL, 0, ~mask);
                 }
+                drm_vblank_post_modeset(dev, radeon_crtc->crtc_id);
+                radeon_crtc_load_lut(crtc);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
+                drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id);
                 if (radeon_crtc->crtc_id)
                         WREG32_P(RADEON_CRTC2_GEN_CNTL, mask, ~mask);
                 else {
@@ -323,10 +326,6 @@ void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
                 }
                 break;
         }
-
-        if (mode != DRM_MODE_DPMS_OFF) {
-                radeon_crtc_load_lut(crtc);
-        }
 }
 
 /* properly set crtc bpp when using atombios */
diff --git a/drivers/gpu/drm/radeon/radeon_reg.h b/drivers/gpu/drm/radeon/radeon_reg.h
index e1b618574461..5a098f304edb 100644
--- a/drivers/gpu/drm/radeon/radeon_reg.h
+++ b/drivers/gpu/drm/radeon/radeon_reg.h
@@ -982,12 +982,15 @@
 #       define RS400_TMDS2_PLLRST           (1 << 1)
 
 #define RADEON_GEN_INT_CNTL                 0x0040
+#       define RADEON_CRTC_VBLANK_MASK          (1 << 0)
+#       define RADEON_CRTC2_VBLANK_MASK         (1 << 9)
 #       define RADEON_SW_INT_ENABLE             (1 << 25)
 #define RADEON_GEN_INT_STATUS               0x0044
-#       define RADEON_VSYNC_INT_AK          (1 <<  2)
-#       define RADEON_VSYNC_INT             (1 <<  2)
-#       define RADEON_VSYNC2_INT_AK         (1 <<  6)
-#       define RADEON_VSYNC2_INT            (1 <<  6)
+#       define AVIVO_DISPLAY_INT_STATUS         (1 << 0)
+#       define RADEON_CRTC_VBLANK_STAT          (1 << 0)
+#       define RADEON_CRTC_VBLANK_STAT_ACK      (1 << 0)
+#       define RADEON_CRTC2_VBLANK_STAT         (1 << 9)
+#       define RADEON_CRTC2_VBLANK_STAT_ACK     (1 << 9)
 #       define RADEON_SW_INT_FIRE               (1 << 26)
 #       define RADEON_SW_INT_TEST               (1 << 25)
 #       define RADEON_SW_INT_TEST_ACK           (1 << 25)
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index bbea6dee4a94..7e8ce983a908 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -240,6 +240,88 @@ void rs600_mc_fini(struct radeon_device *rdev)
 
 
 /*
+ * Interrupts
+ */
+int rs600_irq_set(struct radeon_device *rdev)
+{
+        uint32_t tmp = 0;
+        uint32_t mode_int = 0;
+
+        if (rdev->irq.sw_int) {
+                tmp |= RADEON_SW_INT_ENABLE;
+        }
+        if (rdev->irq.crtc_vblank_int[0]) {
+                tmp |= AVIVO_DISPLAY_INT_STATUS;
+                mode_int |= AVIVO_D1MODE_INT_MASK;
+        }
+        if (rdev->irq.crtc_vblank_int[1]) {
+                tmp |= AVIVO_DISPLAY_INT_STATUS;
+                mode_int |= AVIVO_D2MODE_INT_MASK;
+        }
+        WREG32(RADEON_GEN_INT_CNTL, tmp);
+        WREG32(AVIVO_DxMODE_INT_MASK, mode_int);
+        return 0;
+}
+
+static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_int)
+{
+        uint32_t irqs = RREG32(RADEON_GEN_INT_STATUS);
+        uint32_t irq_mask = RADEON_SW_INT_TEST;
+
+        if (irqs & AVIVO_DISPLAY_INT_STATUS) {
+                *r500_disp_int = RREG32(AVIVO_DISP_INTERRUPT_STATUS);
+                if (*r500_disp_int & AVIVO_D1_VBLANK_INTERRUPT) {
+                        WREG32(AVIVO_D1MODE_VBLANK_STATUS, AVIVO_VBLANK_ACK);
+                }
+                if (*r500_disp_int & AVIVO_D2_VBLANK_INTERRUPT) {
+                        WREG32(AVIVO_D2MODE_VBLANK_STATUS, AVIVO_VBLANK_ACK);
+                }
+        } else {
+                *r500_disp_int = 0;
+        }
+
+        if (irqs) {
+                WREG32(RADEON_GEN_INT_STATUS, irqs);
+        }
+        return irqs & irq_mask;
+}
+
+int rs600_irq_process(struct radeon_device *rdev)
+{
+        uint32_t status;
+        uint32_t r500_disp_int;
+
+        status = rs600_irq_ack(rdev, &r500_disp_int);
+        if (!status && !r500_disp_int) {
+                return IRQ_NONE;
+        }
+        while (status || r500_disp_int) {
+                /* SW interrupt */
+                if (status & RADEON_SW_INT_TEST) {
+                        radeon_fence_process(rdev);
+                }
+                /* Vertical blank interrupts */
+                if (r500_disp_int & AVIVO_D1_VBLANK_INTERRUPT) {
+                        drm_handle_vblank(rdev->ddev, 0);
+                }
+                if (r500_disp_int & AVIVO_D2_VBLANK_INTERRUPT) {
+                        drm_handle_vblank(rdev->ddev, 1);
+                }
+                status = rs600_irq_ack(rdev, &r500_disp_int);
+        }
+        return IRQ_HANDLED;
+}
+
+u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc)
+{
+        if (crtc == 0)
+                return RREG32(AVIVO_D1CRTC_FRAME_COUNT);
+        else
+                return RREG32(AVIVO_D2CRTC_FRAME_COUNT);
+}
+
+
+/*
  * Global GPU functions
  */
 void rs600_disable_vga(struct radeon_device *rdev)