12 files changed, 186 insertions, 3 deletions

diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index 6dcde3798b45..64fdae558d36 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -6033,6 +6033,17 @@ void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, 0);
         radeon_ring_write(ring, 1 << vm_id);
 
+        /* wait for the invalidate to complete */
+        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+        radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
+                                 WAIT_REG_MEM_FUNCTION(0) |  /* always */
+                                 WAIT_REG_MEM_ENGINE(0))); /* me */
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0); /* ref */
+        radeon_ring_write(ring, 0); /* mask */
+        radeon_ring_write(ring, 0x20); /* poll interval */
+
         /* compute doesn't have PFP */
         if (usepfp) {
                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
diff --git a/drivers/gpu/drm/radeon/cik_sdma.c b/drivers/gpu/drm/radeon/cik_sdma.c
index dde5c7e29eb2..a0133c74f4cf 100644
--- a/drivers/gpu/drm/radeon/cik_sdma.c
+++ b/drivers/gpu/drm/radeon/cik_sdma.c
@@ -903,6 +903,9 @@ void cik_sdma_vm_pad_ib(struct radeon_ib *ib)
 void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
                       unsigned vm_id, uint64_t pd_addr)
 {
+        u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
+                          SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
+
         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
         if (vm_id < 8) {
                 radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
@@ -943,5 +946,12 @@ void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
         radeon_ring_write(ring, 1 << vm_id);
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0); /* reference */
+        radeon_ring_write(ring, 0); /* mask */
+        radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
 }
 
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 360de9f1f491..aea48c89b241 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -2516,6 +2516,16 @@ void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
         radeon_ring_write(ring, 1 << vm_id);
 
+        /* wait for the invalidate to complete */
+        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+        radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
+                                 WAIT_REG_MEM_ENGINE(0))); /* me */
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0); /* ref */
+        radeon_ring_write(ring, 0); /* mask */
+        radeon_ring_write(ring, 0x20); /* poll interval */
+
         /* sync PFP to ME, otherwise we might get invalid PFP reads */
         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
         radeon_ring_write(ring, 0x0);
diff --git a/drivers/gpu/drm/radeon/ni_dma.c b/drivers/gpu/drm/radeon/ni_dma.c
index 50f88611ff60..4be2bb7cbef3 100644
--- a/drivers/gpu/drm/radeon/ni_dma.c
+++ b/drivers/gpu/drm/radeon/ni_dma.c
@@ -463,5 +463,11 @@ void cayman_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
         radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
         radeon_ring_write(ring, 1 << vm_id);
+
+        /* wait for invalidate to complete */
+        radeon_ring_write(ring, DMA_SRBM_READ_PACKET);
+        radeon_ring_write(ring, (0xff << 20) | (VM_INVALIDATE_REQUEST >> 2));
+        radeon_ring_write(ring, 0); /* mask */
+        radeon_ring_write(ring, 0); /* value */
 }
 
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
index 2e12e4d69253..ad7125486894 100644
--- a/drivers/gpu/drm/radeon/nid.h
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -1133,6 +1133,23 @@
 #define PACKET3_MEM_SEMAPHORE                           0x39
 #define PACKET3_MPEG_INDEX                              0x3A
 #define PACKET3_WAIT_REG_MEM                            0x3C
+#define         WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+                /* 0 - always
+                 * 1 - <
+                 * 2 - <=
+                 * 3 - ==
+                 * 4 - !=
+                 * 5 - >=
+                 * 6 - >
+                 */
+#define         WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+                /* 0 - reg
+                 * 1 - mem
+                 */
+#define         WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
+                /* 0 - me
+                 * 1 - pfp
+                 */
 #define PACKET3_MEM_WRITE                               0x3D
 #define PACKET3_PFP_SYNC_ME                             0x42
 #define PACKET3_SURFACE_SYNC                            0x43
@@ -1272,6 +1289,13 @@
                                          (1 << 21) |                    \
                                          (((n) & 0xFFFFF) << 0))
 
+#define DMA_SRBM_POLL_PACKET            ((9 << 28) |                    \
+                                         (1 << 27) |                    \
+                                         (1 << 26))
+
+#define DMA_SRBM_READ_PACKET            ((9 << 28) |                    \
+                                         (1 << 27))
+
 /* async DMA Packet types */
 #define DMA_PACKET_WRITE                                  0x2
 #define DMA_PACKET_COPY                                   0x3
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 850de57069be..121aff6a3b41 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -333,6 +333,20 @@ static struct radeon_asic_ring r300_gfx_ring = {
         .set_wptr = &r100_gfx_set_wptr,
 };
 
+static struct radeon_asic_ring rv515_gfx_ring = {
+        .ib_execute = &r100_ring_ib_execute,
+        .emit_fence = &r300_fence_ring_emit,
+        .emit_semaphore = &r100_semaphore_ring_emit,
+        .cs_parse = &r300_cs_parse,
+        .ring_start = &rv515_ring_start,
+        .ring_test = &r100_ring_test,
+        .ib_test = &r100_ib_test,
+        .is_lockup = &r100_gpu_is_lockup,
+        .get_rptr = &r100_gfx_get_rptr,
+        .get_wptr = &r100_gfx_get_wptr,
+        .set_wptr = &r100_gfx_set_wptr,
+};
+
 static struct radeon_asic r300_asic = {
         .init = &r300_init,
         .fini = &r300_fini,
@@ -748,7 +762,7 @@ static struct radeon_asic rv515_asic = {
                 .set_page = &rv370_pcie_gart_set_page,
         },
         .ring = {
-                [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring
+                [RADEON_RING_TYPE_GFX_INDEX] = &rv515_gfx_ring
         },
         .irq = {
                 .set = &rs600_irq_set,
@@ -814,7 +828,7 @@ static struct radeon_asic r520_asic = {
                 .set_page = &rv370_pcie_gart_set_page,
         },
         .ring = {
-                [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring
+                [RADEON_RING_TYPE_GFX_INDEX] = &rv515_gfx_ring
         },
         .irq = {
                 .set = &rs600_irq_set,
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index a46f73737994..d0b4f7d1140d 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -576,7 +576,7 @@ error_unreserve:
 error_free:
         drm_free_large(vm_bos);
 
-        if (r)
+        if (r && r != -ERESTARTSYS)
                 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 32522cc940a1..f7da8fe96a66 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -1287,8 +1287,39 @@ dpm_failed:
         return ret;
 }
 
+struct radeon_dpm_quirk {
+        u32 chip_vendor;
+        u32 chip_device;
+        u32 subsys_vendor;
+        u32 subsys_device;
+};
+
+/* cards with dpm stability problems */
+static struct radeon_dpm_quirk radeon_dpm_quirk_list[] = {
+        /* TURKS - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1386534 */
+        { PCI_VENDOR_ID_ATI, 0x6759, 0x1682, 0x3195 },
+        /* TURKS - https://bugzilla.kernel.org/show_bug.cgi?id=83731 */
+        { PCI_VENDOR_ID_ATI, 0x6840, 0x1179, 0xfb81 },
+        { 0, 0, 0, 0 },
+};
+
 int radeon_pm_init(struct radeon_device *rdev)
 {
+        struct radeon_dpm_quirk *p = radeon_dpm_quirk_list;
+        bool disable_dpm = false;
+
+        /* Apply dpm quirks */
+        while (p && p->chip_device != 0) {
+                if (rdev->pdev->vendor == p->chip_vendor &&
+                    rdev->pdev->device == p->chip_device &&
+                    rdev->pdev->subsystem_vendor == p->subsys_vendor &&
+                    rdev->pdev->subsystem_device == p->subsys_device) {
+                        disable_dpm = true;
+                        break;
+                }
+                ++p;
+        }
+
         /* enable dpm on rv6xx+ */
         switch (rdev->family) {
         case CHIP_RV610:
@@ -1344,6 +1375,8 @@ int radeon_pm_init(struct radeon_device *rdev)
                          (!(rdev->flags & RADEON_IS_IGP)) &&
                          (!rdev->smc_fw))
                         rdev->pm.pm_method = PM_METHOD_PROFILE;
+                else if (disable_dpm && (radeon_dpm == -1))
+                        rdev->pm.pm_method = PM_METHOD_PROFILE;
                 else if (radeon_dpm == 0)
                         rdev->pm.pm_method = PM_METHOD_PROFILE;
                 else
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index 60df444bd075..5d89b874a1a2 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -5057,6 +5057,16 @@ void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, 0);
         radeon_ring_write(ring, 1 << vm_id);
 
+        /* wait for the invalidate to complete */
+        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+        radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
+                                 WAIT_REG_MEM_ENGINE(0))); /* me */
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0); /* ref */
+        radeon_ring_write(ring, 0); /* mask */
+        radeon_ring_write(ring, 0x20); /* poll interval */
+
         /* sync PFP to ME, otherwise we might get invalid PFP reads */
         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
         radeon_ring_write(ring, 0x0);
diff --git a/drivers/gpu/drm/radeon/si_dma.c b/drivers/gpu/drm/radeon/si_dma.c
index f5cc777e1c5f..aa7b872b2c43 100644
--- a/drivers/gpu/drm/radeon/si_dma.c
+++ b/drivers/gpu/drm/radeon/si_dma.c
@@ -206,6 +206,14 @@ void si_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
         radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
         radeon_ring_write(ring, 1 << vm_id);
+
+        /* wait for invalidate to complete */
+        radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST);
+        radeon_ring_write(ring, 0xff << 16); /* retry */
+        radeon_ring_write(ring, 1 << vm_id); /* mask */
+        radeon_ring_write(ring, 0); /* value */
+        radeon_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */
 }
 
 /**
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index 32e354b8b0ab..eff8a6444956 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -2908,6 +2908,22 @@ static int si_init_smc_spll_table(struct radeon_device *rdev)
         return ret;
 }
 
+struct si_dpm_quirk {
+        u32 chip_vendor;
+        u32 chip_device;
+        u32 subsys_vendor;
+        u32 subsys_device;
+        u32 max_sclk;
+        u32 max_mclk;
+};
+
+/* cards with dpm stability problems */
+static struct si_dpm_quirk si_dpm_quirk_list[] = {
+        /* PITCAIRN - https://bugs.freedesktop.org/show_bug.cgi?id=76490 */
+        { PCI_VENDOR_ID_ATI, 0x6810, 0x1462, 0x3036, 0, 120000 },
+        { 0, 0, 0, 0 },
+};
+
 static void si_apply_state_adjust_rules(struct radeon_device *rdev,
                                         struct radeon_ps *rps)
 {
@@ -2918,7 +2934,22 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
         u32 mclk, sclk;
         u16 vddc, vddci;
         u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc;
+        u32 max_sclk = 0, max_mclk = 0;
         int i;
+        struct si_dpm_quirk *p = si_dpm_quirk_list;
+
+        /* Apply dpm quirks */
+        while (p && p->chip_device != 0) {
+                if (rdev->pdev->vendor == p->chip_vendor &&
+                    rdev->pdev->device == p->chip_device &&
+                    rdev->pdev->subsystem_vendor == p->subsys_vendor &&
+                    rdev->pdev->subsystem_device == p->subsys_device) {
+                        max_sclk = p->max_sclk;
+                        max_mclk = p->max_mclk;
+                        break;
+                }
+                ++p;
+        }
 
         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
             ni_dpm_vblank_too_short(rdev))
@@ -2972,6 +3003,14 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
                         if (ps->performance_levels[i].mclk > max_mclk_vddc)
                                 ps->performance_levels[i].mclk = max_mclk_vddc;
                 }
+                if (max_mclk) {
+                        if (ps->performance_levels[i].mclk > max_mclk)
+                                ps->performance_levels[i].mclk = max_mclk;
+                }
+                if (max_sclk) {
+                        if (ps->performance_levels[i].sclk > max_sclk)
+                                ps->performance_levels[i].sclk = max_sclk;
+                }
         }
 
         /* XXX validate the min clocks required for display */
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index 4069be89e585..84999242c747 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -1632,6 +1632,23 @@
 #define PACKET3_MPEG_INDEX                              0x3A
 #define PACKET3_COPY_DW                                 0x3B
 #define PACKET3_WAIT_REG_MEM                            0x3C
+#define         WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+                /* 0 - always
+                 * 1 - <
+                 * 2 - <=
+                 * 3 - ==
+                 * 4 - !=
+                 * 5 - >=
+                 * 6 - >
+                 */
+#define         WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+                /* 0 - reg
+                 * 1 - mem
+                 */
+#define         WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
+                /* 0 - me
+                 * 1 - pfp
+                 */
 #define PACKET3_MEM_WRITE                               0x3D
 #define PACKET3_COPY_DATA                               0x40
 #define PACKET3_CP_DMA                                  0x41
@@ -1835,6 +1852,7 @@
 #define DMA_PACKET_TRAP                                   0x7
 #define DMA_PACKET_SRBM_WRITE                             0x9
 #define DMA_PACKET_CONSTANT_FILL                          0xd
+#define DMA_PACKET_POLL_REG_MEM                           0xe
 #define DMA_PACKET_NOP                                    0xf
 
 #define VCE_STATUS                                      0x20004