drm/radeon/kms: add support for per-ring fence interrupts

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Christian König <deathsimple@vodafone.de>
Signed-off-by: Dave Airlie <airlied@redhat.com>

9 files changed, 105 insertions, 39 deletions

diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 6ff1180c7321..266d411c6d2b 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -40,6 +40,8 @@
 static void evergreen_gpu_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
+extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
+                                     int ring, u32 cp_int_cntl);
 
 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
 {
@@ -2474,7 +2476,13 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev)
 {
         u32 tmp;
 
-        WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+        if (rdev->family >= CHIP_CAYMAN) {
+                cayman_cp_int_cntl_setup(rdev, 0,
+                                         CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+                cayman_cp_int_cntl_setup(rdev, 1, 0);
+                cayman_cp_int_cntl_setup(rdev, 2, 0);
+        } else
+                WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
         WREG32(GRBM_INT_CNTL, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
@@ -2519,6 +2527,7 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev)
 int evergreen_irq_set(struct radeon_device *rdev)
 {
         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
+        u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
         u32 grbm_int_cntl = 0;
@@ -2543,11 +2552,28 @@ int evergreen_irq_set(struct radeon_device *rdev)
         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
 
-        if (rdev->irq.sw_int) {
-                DRM_DEBUG("evergreen_irq_set: sw int\n");
-                cp_int_cntl |= RB_INT_ENABLE;
-                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
+        if (rdev->family >= CHIP_CAYMAN) {
+                /* enable CP interrupts on all rings */
+                if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
+                        DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
+                        cp_int_cntl |= TIME_STAMP_INT_ENABLE;
+                }
+                if (rdev->irq.sw_int[CAYMAN_RING_TYPE_CP1_INDEX]) {
+                        DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
+                        cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
+                }
+                if (rdev->irq.sw_int[CAYMAN_RING_TYPE_CP2_INDEX]) {
+                        DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
+                        cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
+                }
+        } else {
+                if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
+                        DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
+                        cp_int_cntl |= RB_INT_ENABLE;
+                        cp_int_cntl |= TIME_STAMP_INT_ENABLE;
+                }
         }
+
         if (rdev->irq.crtc_vblank_int[0] ||
             rdev->irq.pflip[0]) {
                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
@@ -2607,7 +2633,12 @@ int evergreen_irq_set(struct radeon_device *rdev)
                 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
         }
 
-        WREG32(CP_INT_CNTL, cp_int_cntl);
+        if (rdev->family >= CHIP_CAYMAN) {
+                cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
+                cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
+                cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
+        } else
+                WREG32(CP_INT_CNTL, cp_int_cntl);
         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
 
         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
@@ -3026,7 +3057,20 @@ restart_ih:
                         break;
                 case 181: /* CP EOP event */
                         DRM_DEBUG("IH: CP EOP\n");
-                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
+                        if (rdev->family >= CHIP_CAYMAN) {
+                                switch (src_data) {
+                                case 0:
+                                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
+                                        break;
+                                case 1:
+                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
+                                        break;
+                                case 2:
+                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
+                                        break;
+                                }
+                        } else
+                                radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
                         break;
                 case 233: /* GUI IDLE */
                         DRM_DEBUG("IH: GUI idle\n");
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 26d066442934..30562622b94a 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -1006,6 +1006,15 @@ void cayman_pcie_gart_fini(struct radeon_device *rdev)
         radeon_gart_fini(rdev);
 }
 
+void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
+                              int ring, u32 cp_int_cntl)
+{
+        u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
+
+        WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
+        WREG32(CP_INT_CNTL, cp_int_cntl);
+}
+
 /*
  * CP.
  */
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
index e8d8124834b3..0d3f52cff2f6 100644
--- a/drivers/gpu/drm/radeon/nid.h
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -42,6 +42,9 @@
 #define CAYMAN_MAX_TCC_MASK          0xFF
 
 #define DMIF_ADDR_CONFIG                                0xBD4
+#define SRBM_GFX_CNTL                                   0x0E44
+#define         RINGID(x)                                       (((x) & 0x3) << 0)
+#define         VMID(x)                                         (((x) & 0x7) << 0)
 #define SRBM_STATUS                                     0x0E50
 
 #define VM_CONTEXT0_REQUEST_RESPONSE                    0x1470
@@ -394,6 +397,12 @@
 #define CP_RB0_RPTR_ADDR                                0xC10C
 #define CP_RB0_RPTR_ADDR_HI                             0xC110
 #define CP_RB0_WPTR                                     0xC114
+
+#define CP_INT_CNTL                                     0xC124
+#       define CNTX_BUSY_INT_ENABLE                     (1 << 19)
+#       define CNTX_EMPTY_INT_ENABLE                    (1 << 20)
+#       define TIME_STAMP_INT_ENABLE                    (1 << 26)
+
 #define CP_RB1_BASE                                     0xC180
 #define CP_RB1_CNTL                                     0xC184
 #define CP_RB1_RPTR_ADDR                                0xC188
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index 3ef2c58b8407..d7fd5aa47053 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -667,7 +667,7 @@ int r100_irq_set(struct radeon_device *rdev)
                 WREG32(R_000040_GEN_INT_CNTL, 0);
                 return -EINVAL;
         }
-        if (rdev->irq.sw_int) {
+        if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
                 tmp |= RADEON_SW_INT_ENABLE;
         }
         if (rdev->irq.gui_idle) {
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 48bd8202b5c9..0f39cc661a7e 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -3098,7 +3098,7 @@ int r600_irq_set(struct radeon_device *rdev)
                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
         }
 
-        if (rdev->irq.sw_int) {
+        if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
                 DRM_DEBUG("r600_irq_set: sw int\n");
                 cp_int_cntl |= RB_INT_ENABLE;
                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index e5d5271d7a9c..b4c2d0fe34e3 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -107,6 +107,17 @@ extern int radeon_msi;
 #define RADEONFB_CONN_LIMIT             4
 #define RADEON_BIOS_NUM_SCRATCH         8
 
+/* max number of rings */
+#define RADEON_NUM_RINGS 3
+
+/* internal ring indices */
+/* r1xx+ has gfx CP ring */
+#define RADEON_RING_TYPE_GFX_INDEX  0
+
+/* cayman has 2 compute CP rings */
+#define CAYMAN_RING_TYPE_CP1_INDEX 1
+#define CAYMAN_RING_TYPE_CP2_INDEX 2
+
 /*
  * Errata workarounds.
  */
@@ -464,7 +475,7 @@ union radeon_irq_stat_regs {
 
 struct radeon_irq {
         bool            installed;
-        bool            sw_int;
+        bool            sw_int[RADEON_NUM_RINGS];
         bool            crtc_vblank_int[RADEON_MAX_CRTCS];
         bool            pflip[RADEON_MAX_CRTCS];
         wait_queue_head_t       vblank_queue;
@@ -474,7 +485,7 @@ struct radeon_irq {
         wait_queue_head_t       idle_queue;
         bool            hdmi[RADEON_MAX_HDMI_BLOCKS];
         spinlock_t sw_lock;
-        int sw_refcount;
+        int sw_refcount[RADEON_NUM_RINGS];
         union radeon_irq_stat_regs stat_regs;
         spinlock_t pflip_lock[RADEON_MAX_CRTCS];
         int pflip_refcount[RADEON_MAX_CRTCS];
@@ -482,8 +493,8 @@ struct radeon_irq {
 
 int radeon_irq_kms_init(struct radeon_device *rdev);
 void radeon_irq_kms_fini(struct radeon_device *rdev);
-void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev);
-void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev);
+void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring);
+void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring);
 void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
 void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
 
@@ -491,17 +502,6 @@ void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
  * CP & rings.
  */
 
-/* max number of rings */
-#define RADEON_NUM_RINGS 3
-
-/* internal ring indices */
-/* r1xx+ has gfx CP ring */
-#define RADEON_RING_TYPE_GFX_INDEX  0
-
-/* cayman has 2 compute CP rings */
-#define CAYMAN_RING_TYPE_CP1_INDEX 1
-#define CAYMAN_RING_TYPE_CP2_INDEX 2
-
 struct radeon_ib {
         struct list_head        list;
         unsigned                idx;
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 8d626baaa064..ae9e3da594a1 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -230,18 +230,18 @@ retry:
         seq = rdev->fence_drv[fence->ring].last_seq;
         trace_radeon_fence_wait_begin(rdev->ddev, seq);
         if (intr) {
-                radeon_irq_kms_sw_irq_get(rdev);
+                radeon_irq_kms_sw_irq_get(rdev, fence->ring);
                 r = wait_event_interruptible_timeout(rdev->fence_drv[fence->ring].queue,
                                 radeon_fence_signaled(fence), timeout);
-                radeon_irq_kms_sw_irq_put(rdev);
+                radeon_irq_kms_sw_irq_put(rdev, fence->ring);
                 if (unlikely(r < 0)) {
                         return r;
                 }
         } else {
-                radeon_irq_kms_sw_irq_get(rdev);
+                radeon_irq_kms_sw_irq_get(rdev, fence->ring);
                 r = wait_event_timeout(rdev->fence_drv[fence->ring].queue,
                          radeon_fence_signaled(fence), timeout);
-                radeon_irq_kms_sw_irq_put(rdev);
+                radeon_irq_kms_sw_irq_put(rdev, fence->ring);
         }
         trace_radeon_fence_wait_end(rdev->ddev, seq);
         if (unlikely(!radeon_fence_signaled(fence))) {
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 8f86aeb26693..be38921bf761 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -65,7 +65,8 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
         unsigned i;
 
         /* Disable *all* interrupts */
-        rdev->irq.sw_int = false;
+        for (i = 0; i < RADEON_NUM_RINGS; i++)
+                rdev->irq.sw_int[i] = false;
         rdev->irq.gui_idle = false;
         for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
                 rdev->irq.hpd[i] = false;
@@ -81,9 +82,11 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
 int radeon_driver_irq_postinstall_kms(struct drm_device *dev)
 {
         struct radeon_device *rdev = dev->dev_private;
+        unsigned i;
 
         dev->max_vblank_count = 0x001fffff;
-        rdev->irq.sw_int = true;
+        for (i = 0; i < RADEON_NUM_RINGS; i++)
+                rdev->irq.sw_int[i] = true;
         radeon_irq_set(rdev);
         return 0;
 }
@@ -97,7 +100,8 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
                 return;
         }
         /* Disable *all* interrupts */
-        rdev->irq.sw_int = false;
+        for (i = 0; i < RADEON_NUM_RINGS; i++)
+                rdev->irq.sw_int[i] = false;
         rdev->irq.gui_idle = false;
         for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
                 rdev->irq.hpd[i] = false;
@@ -194,26 +198,26 @@ void radeon_irq_kms_fini(struct radeon_device *rdev)
         flush_work_sync(&rdev->hotplug_work);
 }
 
-void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev)
+void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring)
 {
         unsigned long irqflags;
 
         spin_lock_irqsave(&rdev->irq.sw_lock, irqflags);
-        if (rdev->ddev->irq_enabled && (++rdev->irq.sw_refcount == 1)) {
-                rdev->irq.sw_int = true;
+        if (rdev->ddev->irq_enabled && (++rdev->irq.sw_refcount[ring] == 1)) {
+                rdev->irq.sw_int[ring] = true;
                 radeon_irq_set(rdev);
         }
         spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags);
 }
 
-void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev)
+void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring)
 {
         unsigned long irqflags;
 
         spin_lock_irqsave(&rdev->irq.sw_lock, irqflags);
-        BUG_ON(rdev->ddev->irq_enabled && rdev->irq.sw_refcount <= 0);
-        if (rdev->ddev->irq_enabled && (--rdev->irq.sw_refcount == 0)) {
-                rdev->irq.sw_int = false;
+        BUG_ON(rdev->ddev->irq_enabled && rdev->irq.sw_refcount[ring] <= 0);
+        if (rdev->ddev->irq_enabled && (--rdev->irq.sw_refcount[ring] == 0)) {
+                rdev->irq.sw_int[ring] = false;
                 radeon_irq_set(rdev);
         }
         spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags);
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index 3fe384741fc0..8a52cf007ff0 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -549,7 +549,7 @@ int rs600_irq_set(struct radeon_device *rdev)
                 WREG32(R_000040_GEN_INT_CNTL, 0);
                 return -EINVAL;
         }
-        if (rdev->irq.sw_int) {
+        if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
                 tmp |= S_000040_SW_INT_EN(1);
         }
         if (rdev->irq.gui_idle) {