aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/intel_ringbuffer.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/intel_ringbuffer.c')
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c926
1 files changed, 372 insertions, 554 deletions
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index fedd27049814..61e00bf9e87f 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -58,18 +58,10 @@ void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
58 ringbuf->tail, ringbuf->size); 58 ringbuf->tail, ringbuf->size);
59} 59}
60 60
61bool intel_engine_stopped(struct intel_engine_cs *engine)
62{
63 struct drm_i915_private *dev_priv = engine->i915;
64 return dev_priv->gpu_error.stop_rings & intel_engine_flag(engine);
65}
66
67static void __intel_ring_advance(struct intel_engine_cs *engine) 61static void __intel_ring_advance(struct intel_engine_cs *engine)
68{ 62{
69 struct intel_ringbuffer *ringbuf = engine->buffer; 63 struct intel_ringbuffer *ringbuf = engine->buffer;
70 ringbuf->tail &= ringbuf->size - 1; 64 ringbuf->tail &= ringbuf->size - 1;
71 if (intel_engine_stopped(engine))
72 return;
73 engine->write_tail(engine, ringbuf->tail); 65 engine->write_tail(engine, ringbuf->tail);
74} 66}
75 67
@@ -515,8 +507,9 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
515 I915_WRITE(reg, 507 I915_WRITE(reg,
516 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 508 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
517 INSTPM_SYNC_FLUSH)); 509 INSTPM_SYNC_FLUSH));
518 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, 510 if (intel_wait_for_register(dev_priv,
519 1000)) 511 reg, INSTPM_SYNC_FLUSH, 0,
512 1000))
520 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", 513 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
521 engine->name); 514 engine->name);
522 } 515 }
@@ -528,7 +521,11 @@ static bool stop_ring(struct intel_engine_cs *engine)
528 521
529 if (!IS_GEN2(dev_priv)) { 522 if (!IS_GEN2(dev_priv)) {
530 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); 523 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
531 if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) { 524 if (intel_wait_for_register(dev_priv,
525 RING_MI_MODE(engine->mmio_base),
526 MODE_IDLE,
527 MODE_IDLE,
528 1000)) {
532 DRM_ERROR("%s : timed out trying to stop ring\n", 529 DRM_ERROR("%s : timed out trying to stop ring\n",
533 engine->name); 530 engine->name);
534 /* Sometimes we observe that the idle flag is not 531 /* Sometimes we observe that the idle flag is not
@@ -643,58 +640,42 @@ out:
643 return ret; 640 return ret;
644} 641}
645 642
646void 643void intel_fini_pipe_control(struct intel_engine_cs *engine)
647intel_fini_pipe_control(struct intel_engine_cs *engine)
648{ 644{
649 if (engine->scratch.obj == NULL) 645 if (engine->scratch.obj == NULL)
650 return; 646 return;
651 647
652 if (INTEL_GEN(engine->i915) >= 5) { 648 i915_gem_object_ggtt_unpin(engine->scratch.obj);
653 kunmap(sg_page(engine->scratch.obj->pages->sgl));
654 i915_gem_object_ggtt_unpin(engine->scratch.obj);
655 }
656
657 drm_gem_object_unreference(&engine->scratch.obj->base); 649 drm_gem_object_unreference(&engine->scratch.obj->base);
658 engine->scratch.obj = NULL; 650 engine->scratch.obj = NULL;
659} 651}
660 652
661int 653int intel_init_pipe_control(struct intel_engine_cs *engine, int size)
662intel_init_pipe_control(struct intel_engine_cs *engine)
663{ 654{
655 struct drm_i915_gem_object *obj;
664 int ret; 656 int ret;
665 657
666 WARN_ON(engine->scratch.obj); 658 WARN_ON(engine->scratch.obj);
667 659
668 engine->scratch.obj = i915_gem_object_create(engine->i915->dev, 4096); 660 obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
669 if (IS_ERR(engine->scratch.obj)) { 661 if (!obj)
670 DRM_ERROR("Failed to allocate seqno page\n"); 662 obj = i915_gem_object_create(&engine->i915->drm, size);
671 ret = PTR_ERR(engine->scratch.obj); 663 if (IS_ERR(obj)) {
672 engine->scratch.obj = NULL; 664 DRM_ERROR("Failed to allocate scratch page\n");
665 ret = PTR_ERR(obj);
673 goto err; 666 goto err;
674 } 667 }
675 668
676 ret = i915_gem_object_set_cache_level(engine->scratch.obj, 669 ret = i915_gem_obj_ggtt_pin(obj, 4096, PIN_HIGH);
677 I915_CACHE_LLC);
678 if (ret) 670 if (ret)
679 goto err_unref; 671 goto err_unref;
680 672
681 ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0); 673 engine->scratch.obj = obj;
682 if (ret) 674 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
683 goto err_unref;
684
685 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj);
686 engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl));
687 if (engine->scratch.cpu_page == NULL) {
688 ret = -ENOMEM;
689 goto err_unpin;
690 }
691
692 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n", 675 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
693 engine->name, engine->scratch.gtt_offset); 676 engine->name, engine->scratch.gtt_offset);
694 return 0; 677 return 0;
695 678
696err_unpin:
697 i915_gem_object_ggtt_unpin(engine->scratch.obj);
698err_unref: 679err_unref:
699 drm_gem_object_unreference(&engine->scratch.obj->base); 680 drm_gem_object_unreference(&engine->scratch.obj->base);
700err: 681err:
@@ -1324,8 +1305,7 @@ static int init_render_ring(struct intel_engine_cs *engine)
1324 if (IS_GEN(dev_priv, 6, 7)) 1305 if (IS_GEN(dev_priv, 6, 7))
1325 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 1306 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1326 1307
1327 if (HAS_L3_DPF(dev_priv)) 1308 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1328 I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev_priv));
1329 1309
1330 return init_workarounds_ring(engine); 1310 return init_workarounds_ring(engine);
1331} 1311}
@@ -1362,19 +1342,17 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
1362 return ret; 1342 return ret;
1363 1343
1364 for_each_engine_id(waiter, dev_priv, id) { 1344 for_each_engine_id(waiter, dev_priv, id) {
1365 u32 seqno;
1366 u64 gtt_offset = signaller->semaphore.signal_ggtt[id]; 1345 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1367 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) 1346 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1368 continue; 1347 continue;
1369 1348
1370 seqno = i915_gem_request_get_seqno(signaller_req);
1371 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6)); 1349 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
1372 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB | 1350 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
1373 PIPE_CONTROL_QW_WRITE | 1351 PIPE_CONTROL_QW_WRITE |
1374 PIPE_CONTROL_CS_STALL); 1352 PIPE_CONTROL_CS_STALL);
1375 intel_ring_emit(signaller, lower_32_bits(gtt_offset)); 1353 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
1376 intel_ring_emit(signaller, upper_32_bits(gtt_offset)); 1354 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1377 intel_ring_emit(signaller, seqno); 1355 intel_ring_emit(signaller, signaller_req->seqno);
1378 intel_ring_emit(signaller, 0); 1356 intel_ring_emit(signaller, 0);
1379 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | 1357 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1380 MI_SEMAPHORE_TARGET(waiter->hw_id)); 1358 MI_SEMAPHORE_TARGET(waiter->hw_id));
@@ -1403,18 +1381,16 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
1403 return ret; 1381 return ret;
1404 1382
1405 for_each_engine_id(waiter, dev_priv, id) { 1383 for_each_engine_id(waiter, dev_priv, id) {
1406 u32 seqno;
1407 u64 gtt_offset = signaller->semaphore.signal_ggtt[id]; 1384 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1408 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) 1385 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1409 continue; 1386 continue;
1410 1387
1411 seqno = i915_gem_request_get_seqno(signaller_req);
1412 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) | 1388 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
1413 MI_FLUSH_DW_OP_STOREDW); 1389 MI_FLUSH_DW_OP_STOREDW);
1414 intel_ring_emit(signaller, lower_32_bits(gtt_offset) | 1390 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
1415 MI_FLUSH_DW_USE_GTT); 1391 MI_FLUSH_DW_USE_GTT);
1416 intel_ring_emit(signaller, upper_32_bits(gtt_offset)); 1392 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1417 intel_ring_emit(signaller, seqno); 1393 intel_ring_emit(signaller, signaller_req->seqno);
1418 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | 1394 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1419 MI_SEMAPHORE_TARGET(waiter->hw_id)); 1395 MI_SEMAPHORE_TARGET(waiter->hw_id));
1420 intel_ring_emit(signaller, 0); 1396 intel_ring_emit(signaller, 0);
@@ -1445,11 +1421,9 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req,
1445 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id]; 1421 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id];
1446 1422
1447 if (i915_mmio_reg_valid(mbox_reg)) { 1423 if (i915_mmio_reg_valid(mbox_reg)) {
1448 u32 seqno = i915_gem_request_get_seqno(signaller_req);
1449
1450 intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); 1424 intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
1451 intel_ring_emit_reg(signaller, mbox_reg); 1425 intel_ring_emit_reg(signaller, mbox_reg);
1452 intel_ring_emit(signaller, seqno); 1426 intel_ring_emit(signaller, signaller_req->seqno);
1453 } 1427 }
1454 } 1428 }
1455 1429
@@ -1485,7 +1459,7 @@ gen6_add_request(struct drm_i915_gem_request *req)
1485 intel_ring_emit(engine, MI_STORE_DWORD_INDEX); 1459 intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1486 intel_ring_emit(engine, 1460 intel_ring_emit(engine,
1487 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 1461 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1488 intel_ring_emit(engine, i915_gem_request_get_seqno(req)); 1462 intel_ring_emit(engine, req->seqno);
1489 intel_ring_emit(engine, MI_USER_INTERRUPT); 1463 intel_ring_emit(engine, MI_USER_INTERRUPT);
1490 __intel_ring_advance(engine); 1464 __intel_ring_advance(engine);
1491 1465
@@ -1542,6 +1516,7 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
1542{ 1516{
1543 struct intel_engine_cs *waiter = waiter_req->engine; 1517 struct intel_engine_cs *waiter = waiter_req->engine;
1544 struct drm_i915_private *dev_priv = waiter_req->i915; 1518 struct drm_i915_private *dev_priv = waiter_req->i915;
1519 u64 offset = GEN8_WAIT_OFFSET(waiter, signaller->id);
1545 struct i915_hw_ppgtt *ppgtt; 1520 struct i915_hw_ppgtt *ppgtt;
1546 int ret; 1521 int ret;
1547 1522
@@ -1553,10 +1528,8 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
1553 MI_SEMAPHORE_GLOBAL_GTT | 1528 MI_SEMAPHORE_GLOBAL_GTT |
1554 MI_SEMAPHORE_SAD_GTE_SDD); 1529 MI_SEMAPHORE_SAD_GTE_SDD);
1555 intel_ring_emit(waiter, seqno); 1530 intel_ring_emit(waiter, seqno);
1556 intel_ring_emit(waiter, 1531 intel_ring_emit(waiter, lower_32_bits(offset));
1557 lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); 1532 intel_ring_emit(waiter, upper_32_bits(offset));
1558 intel_ring_emit(waiter,
1559 upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1560 intel_ring_advance(waiter); 1533 intel_ring_advance(waiter);
1561 1534
1562 /* When the !RCS engines idle waiting upon a semaphore, they lose their 1535 /* When the !RCS engines idle waiting upon a semaphore, they lose their
@@ -1611,66 +1584,22 @@ gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
1611 return 0; 1584 return 0;
1612} 1585}
1613 1586
1614#define PIPE_CONTROL_FLUSH(ring__, addr__) \ 1587static void
1615do { \ 1588gen5_seqno_barrier(struct intel_engine_cs *ring)
1616 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
1617 PIPE_CONTROL_DEPTH_STALL); \
1618 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
1619 intel_ring_emit(ring__, 0); \
1620 intel_ring_emit(ring__, 0); \
1621} while (0)
1622
1623static int
1624pc_render_add_request(struct drm_i915_gem_request *req)
1625{ 1589{
1626 struct intel_engine_cs *engine = req->engine; 1590 /* MI_STORE are internally buffered by the GPU and not flushed
1627 u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES; 1591 * either by MI_FLUSH or SyncFlush or any other combination of
1628 int ret; 1592 * MI commands.
1629
1630 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
1631 * incoherent with writes to memory, i.e. completely fubar,
1632 * so we need to use PIPE_NOTIFY instead.
1633 * 1593 *
1634 * However, we also need to workaround the qword write 1594 * "Only the submission of the store operation is guaranteed.
1635 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to 1595 * The write result will be complete (coherent) some time later
1636 * memory before requesting an interrupt. 1596 * (this is practically a finite period but there is no guaranteed
1597 * latency)."
1598 *
1599 * Empirically, we observe that we need a delay of at least 75us to
1600 * be sure that the seqno write is visible by the CPU.
1637 */ 1601 */
1638 ret = intel_ring_begin(req, 32); 1602 usleep_range(125, 250);
1639 if (ret)
1640 return ret;
1641
1642 intel_ring_emit(engine,
1643 GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1644 PIPE_CONTROL_WRITE_FLUSH |
1645 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
1646 intel_ring_emit(engine,
1647 engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1648 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1649 intel_ring_emit(engine, 0);
1650 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1651 scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
1652 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1653 scratch_addr += 2 * CACHELINE_BYTES;
1654 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1655 scratch_addr += 2 * CACHELINE_BYTES;
1656 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1657 scratch_addr += 2 * CACHELINE_BYTES;
1658 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1659 scratch_addr += 2 * CACHELINE_BYTES;
1660 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1661
1662 intel_ring_emit(engine,
1663 GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1664 PIPE_CONTROL_WRITE_FLUSH |
1665 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
1666 PIPE_CONTROL_NOTIFY);
1667 intel_ring_emit(engine,
1668 engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1669 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1670 intel_ring_emit(engine, 0);
1671 __intel_ring_advance(engine);
1672
1673 return 0;
1674} 1603}
1675 1604
1676static void 1605static void
@@ -1698,127 +1627,54 @@ gen6_seqno_barrier(struct intel_engine_cs *engine)
1698 spin_unlock_irq(&dev_priv->uncore.lock); 1627 spin_unlock_irq(&dev_priv->uncore.lock);
1699} 1628}
1700 1629
1701static u32
1702ring_get_seqno(struct intel_engine_cs *engine)
1703{
1704 return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
1705}
1706
1707static void 1630static void
1708ring_set_seqno(struct intel_engine_cs *engine, u32 seqno) 1631gen5_irq_enable(struct intel_engine_cs *engine)
1709{
1710 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
1711}
1712
1713static u32
1714pc_render_get_seqno(struct intel_engine_cs *engine)
1715{ 1632{
1716 return engine->scratch.cpu_page[0]; 1633 gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
1717} 1634}
1718 1635
1719static void 1636static void
1720pc_render_set_seqno(struct intel_engine_cs *engine, u32 seqno) 1637gen5_irq_disable(struct intel_engine_cs *engine)
1721{ 1638{
1722 engine->scratch.cpu_page[0] = seqno; 1639 gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
1723}
1724
1725static bool
1726gen5_ring_get_irq(struct intel_engine_cs *engine)
1727{
1728 struct drm_i915_private *dev_priv = engine->i915;
1729 unsigned long flags;
1730
1731 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1732 return false;
1733
1734 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1735 if (engine->irq_refcount++ == 0)
1736 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1737 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1738
1739 return true;
1740} 1640}
1741 1641
1742static void 1642static void
1743gen5_ring_put_irq(struct intel_engine_cs *engine) 1643i9xx_irq_enable(struct intel_engine_cs *engine)
1744{ 1644{
1745 struct drm_i915_private *dev_priv = engine->i915; 1645 struct drm_i915_private *dev_priv = engine->i915;
1746 unsigned long flags;
1747
1748 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1749 if (--engine->irq_refcount == 0)
1750 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1751 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1752}
1753
1754static bool
1755i9xx_ring_get_irq(struct intel_engine_cs *engine)
1756{
1757 struct drm_i915_private *dev_priv = engine->i915;
1758 unsigned long flags;
1759
1760 if (!intel_irqs_enabled(dev_priv))
1761 return false;
1762 1646
1763 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1647 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1764 if (engine->irq_refcount++ == 0) { 1648 I915_WRITE(IMR, dev_priv->irq_mask);
1765 dev_priv->irq_mask &= ~engine->irq_enable_mask; 1649 POSTING_READ_FW(RING_IMR(engine->mmio_base));
1766 I915_WRITE(IMR, dev_priv->irq_mask);
1767 POSTING_READ(IMR);
1768 }
1769 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1770
1771 return true;
1772} 1650}
1773 1651
1774static void 1652static void
1775i9xx_ring_put_irq(struct intel_engine_cs *engine) 1653i9xx_irq_disable(struct intel_engine_cs *engine)
1776{ 1654{
1777 struct drm_i915_private *dev_priv = engine->i915; 1655 struct drm_i915_private *dev_priv = engine->i915;
1778 unsigned long flags;
1779 1656
1780 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1657 dev_priv->irq_mask |= engine->irq_enable_mask;
1781 if (--engine->irq_refcount == 0) { 1658 I915_WRITE(IMR, dev_priv->irq_mask);
1782 dev_priv->irq_mask |= engine->irq_enable_mask;
1783 I915_WRITE(IMR, dev_priv->irq_mask);
1784 POSTING_READ(IMR);
1785 }
1786 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1787} 1659}
1788 1660
1789static bool 1661static void
1790i8xx_ring_get_irq(struct intel_engine_cs *engine) 1662i8xx_irq_enable(struct intel_engine_cs *engine)
1791{ 1663{
1792 struct drm_i915_private *dev_priv = engine->i915; 1664 struct drm_i915_private *dev_priv = engine->i915;
1793 unsigned long flags;
1794
1795 if (!intel_irqs_enabled(dev_priv))
1796 return false;
1797 1665
1798 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1666 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1799 if (engine->irq_refcount++ == 0) { 1667 I915_WRITE16(IMR, dev_priv->irq_mask);
1800 dev_priv->irq_mask &= ~engine->irq_enable_mask; 1668 POSTING_READ16(RING_IMR(engine->mmio_base));
1801 I915_WRITE16(IMR, dev_priv->irq_mask);
1802 POSTING_READ16(IMR);
1803 }
1804 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1805
1806 return true;
1807} 1669}
1808 1670
1809static void 1671static void
1810i8xx_ring_put_irq(struct intel_engine_cs *engine) 1672i8xx_irq_disable(struct intel_engine_cs *engine)
1811{ 1673{
1812 struct drm_i915_private *dev_priv = engine->i915; 1674 struct drm_i915_private *dev_priv = engine->i915;
1813 unsigned long flags;
1814 1675
1815 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1676 dev_priv->irq_mask |= engine->irq_enable_mask;
1816 if (--engine->irq_refcount == 0) { 1677 I915_WRITE16(IMR, dev_priv->irq_mask);
1817 dev_priv->irq_mask |= engine->irq_enable_mask;
1818 I915_WRITE16(IMR, dev_priv->irq_mask);
1819 POSTING_READ16(IMR);
1820 }
1821 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1822} 1678}
1823 1679
1824static int 1680static int
@@ -1852,129 +1708,68 @@ i9xx_add_request(struct drm_i915_gem_request *req)
1852 intel_ring_emit(engine, MI_STORE_DWORD_INDEX); 1708 intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1853 intel_ring_emit(engine, 1709 intel_ring_emit(engine,
1854 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 1710 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1855 intel_ring_emit(engine, i915_gem_request_get_seqno(req)); 1711 intel_ring_emit(engine, req->seqno);
1856 intel_ring_emit(engine, MI_USER_INTERRUPT); 1712 intel_ring_emit(engine, MI_USER_INTERRUPT);
1857 __intel_ring_advance(engine); 1713 __intel_ring_advance(engine);
1858 1714
1859 return 0; 1715 return 0;
1860} 1716}
1861 1717
1862static bool 1718static void
1863gen6_ring_get_irq(struct intel_engine_cs *engine) 1719gen6_irq_enable(struct intel_engine_cs *engine)
1864{ 1720{
1865 struct drm_i915_private *dev_priv = engine->i915; 1721 struct drm_i915_private *dev_priv = engine->i915;
1866 unsigned long flags;
1867
1868 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1869 return false;
1870
1871 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1872 if (engine->irq_refcount++ == 0) {
1873 if (HAS_L3_DPF(dev_priv) && engine->id == RCS)
1874 I915_WRITE_IMR(engine,
1875 ~(engine->irq_enable_mask |
1876 GT_PARITY_ERROR(dev_priv)));
1877 else
1878 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1879 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1880 }
1881 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1882 1722
1883 return true; 1723 I915_WRITE_IMR(engine,
1724 ~(engine->irq_enable_mask |
1725 engine->irq_keep_mask));
1726 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1884} 1727}
1885 1728
1886static void 1729static void
1887gen6_ring_put_irq(struct intel_engine_cs *engine) 1730gen6_irq_disable(struct intel_engine_cs *engine)
1888{ 1731{
1889 struct drm_i915_private *dev_priv = engine->i915; 1732 struct drm_i915_private *dev_priv = engine->i915;
1890 unsigned long flags;
1891 1733
1892 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1734 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1893 if (--engine->irq_refcount == 0) { 1735 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1894 if (HAS_L3_DPF(dev_priv) && engine->id == RCS)
1895 I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev_priv));
1896 else
1897 I915_WRITE_IMR(engine, ~0);
1898 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1899 }
1900 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1901} 1736}
1902 1737
1903static bool 1738static void
1904hsw_vebox_get_irq(struct intel_engine_cs *engine) 1739hsw_vebox_irq_enable(struct intel_engine_cs *engine)
1905{ 1740{
1906 struct drm_i915_private *dev_priv = engine->i915; 1741 struct drm_i915_private *dev_priv = engine->i915;
1907 unsigned long flags;
1908 1742
1909 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 1743 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1910 return false; 1744 gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
1911
1912 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1913 if (engine->irq_refcount++ == 0) {
1914 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1915 gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
1916 }
1917 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1918
1919 return true;
1920} 1745}
1921 1746
1922static void 1747static void
1923hsw_vebox_put_irq(struct intel_engine_cs *engine) 1748hsw_vebox_irq_disable(struct intel_engine_cs *engine)
1924{ 1749{
1925 struct drm_i915_private *dev_priv = engine->i915; 1750 struct drm_i915_private *dev_priv = engine->i915;
1926 unsigned long flags;
1927 1751
1928 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1752 I915_WRITE_IMR(engine, ~0);
1929 if (--engine->irq_refcount == 0) { 1753 gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
1930 I915_WRITE_IMR(engine, ~0);
1931 gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
1932 }
1933 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1934} 1754}
1935 1755
1936static bool 1756static void
1937gen8_ring_get_irq(struct intel_engine_cs *engine) 1757gen8_irq_enable(struct intel_engine_cs *engine)
1938{ 1758{
1939 struct drm_i915_private *dev_priv = engine->i915; 1759 struct drm_i915_private *dev_priv = engine->i915;
1940 unsigned long flags;
1941 1760
1942 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 1761 I915_WRITE_IMR(engine,
1943 return false; 1762 ~(engine->irq_enable_mask |
1944 1763 engine->irq_keep_mask));
1945 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1764 POSTING_READ_FW(RING_IMR(engine->mmio_base));
1946 if (engine->irq_refcount++ == 0) {
1947 if (HAS_L3_DPF(dev_priv) && engine->id == RCS) {
1948 I915_WRITE_IMR(engine,
1949 ~(engine->irq_enable_mask |
1950 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1951 } else {
1952 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1953 }
1954 POSTING_READ(RING_IMR(engine->mmio_base));
1955 }
1956 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1957
1958 return true;
1959} 1765}
1960 1766
1961static void 1767static void
1962gen8_ring_put_irq(struct intel_engine_cs *engine) 1768gen8_irq_disable(struct intel_engine_cs *engine)
1963{ 1769{
1964 struct drm_i915_private *dev_priv = engine->i915; 1770 struct drm_i915_private *dev_priv = engine->i915;
1965 unsigned long flags;
1966 1771
1967 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1772 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1968 if (--engine->irq_refcount == 0) {
1969 if (HAS_L3_DPF(dev_priv) && engine->id == RCS) {
1970 I915_WRITE_IMR(engine,
1971 ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1972 } else {
1973 I915_WRITE_IMR(engine, ~0);
1974 }
1975 POSTING_READ(RING_IMR(engine->mmio_base));
1976 }
1977 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1978} 1773}
1979 1774
1980static int 1775static int
@@ -2093,7 +1888,7 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)
2093 if (!dev_priv->status_page_dmah) 1888 if (!dev_priv->status_page_dmah)
2094 return; 1889 return;
2095 1890
2096 drm_pci_free(dev_priv->dev, dev_priv->status_page_dmah); 1891 drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
2097 engine->status_page.page_addr = NULL; 1892 engine->status_page.page_addr = NULL;
2098} 1893}
2099 1894
@@ -2119,7 +1914,7 @@ static int init_status_page(struct intel_engine_cs *engine)
2119 unsigned flags; 1914 unsigned flags;
2120 int ret; 1915 int ret;
2121 1916
2122 obj = i915_gem_object_create(engine->i915->dev, 4096); 1917 obj = i915_gem_object_create(&engine->i915->drm, 4096);
2123 if (IS_ERR(obj)) { 1918 if (IS_ERR(obj)) {
2124 DRM_ERROR("Failed to allocate status page\n"); 1919 DRM_ERROR("Failed to allocate status page\n");
2125 return PTR_ERR(obj); 1920 return PTR_ERR(obj);
@@ -2168,7 +1963,7 @@ static int init_phys_status_page(struct intel_engine_cs *engine)
2168 1963
2169 if (!dev_priv->status_page_dmah) { 1964 if (!dev_priv->status_page_dmah) {
2170 dev_priv->status_page_dmah = 1965 dev_priv->status_page_dmah =
2171 drm_pci_alloc(dev_priv->dev, PAGE_SIZE, PAGE_SIZE); 1966 drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
2172 if (!dev_priv->status_page_dmah) 1967 if (!dev_priv->status_page_dmah)
2173 return -ENOMEM; 1968 return -ENOMEM;
2174 } 1969 }
@@ -2301,7 +2096,7 @@ intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
2301 ring->last_retired_head = -1; 2096 ring->last_retired_head = -1;
2302 intel_ring_update_space(ring); 2097 intel_ring_update_space(ring);
2303 2098
2304 ret = intel_alloc_ringbuffer_obj(engine->i915->dev, ring); 2099 ret = intel_alloc_ringbuffer_obj(&engine->i915->drm, ring);
2305 if (ret) { 2100 if (ret) {
2306 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n", 2101 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
2307 engine->name, ret); 2102 engine->name, ret);
@@ -2321,6 +2116,57 @@ intel_ringbuffer_free(struct intel_ringbuffer *ring)
2321 kfree(ring); 2116 kfree(ring);
2322} 2117}
2323 2118
2119static int intel_ring_context_pin(struct i915_gem_context *ctx,
2120 struct intel_engine_cs *engine)
2121{
2122 struct intel_context *ce = &ctx->engine[engine->id];
2123 int ret;
2124
2125 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
2126
2127 if (ce->pin_count++)
2128 return 0;
2129
2130 if (ce->state) {
2131 ret = i915_gem_obj_ggtt_pin(ce->state, ctx->ggtt_alignment, 0);
2132 if (ret)
2133 goto error;
2134 }
2135
2136 /* The kernel context is only used as a placeholder for flushing the
2137 * active context. It is never used for submitting user rendering and
2138 * as such never requires the golden render context, and so we can skip
2139 * emitting it when we switch to the kernel context. This is required
2140 * as during eviction we cannot allocate and pin the renderstate in
2141 * order to initialise the context.
2142 */
2143 if (ctx == ctx->i915->kernel_context)
2144 ce->initialised = true;
2145
2146 i915_gem_context_reference(ctx);
2147 return 0;
2148
2149error:
2150 ce->pin_count = 0;
2151 return ret;
2152}
2153
2154static void intel_ring_context_unpin(struct i915_gem_context *ctx,
2155 struct intel_engine_cs *engine)
2156{
2157 struct intel_context *ce = &ctx->engine[engine->id];
2158
2159 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
2160
2161 if (--ce->pin_count)
2162 return;
2163
2164 if (ce->state)
2165 i915_gem_object_ggtt_unpin(ce->state);
2166
2167 i915_gem_context_unreference(ctx);
2168}
2169
2324static int intel_init_ring_buffer(struct drm_device *dev, 2170static int intel_init_ring_buffer(struct drm_device *dev,
2325 struct intel_engine_cs *engine) 2171 struct intel_engine_cs *engine)
2326{ 2172{
@@ -2339,7 +2185,20 @@ static int intel_init_ring_buffer(struct drm_device *dev,
2339 memset(engine->semaphore.sync_seqno, 0, 2185 memset(engine->semaphore.sync_seqno, 0,
2340 sizeof(engine->semaphore.sync_seqno)); 2186 sizeof(engine->semaphore.sync_seqno));
2341 2187
2342 init_waitqueue_head(&engine->irq_queue); 2188 ret = intel_engine_init_breadcrumbs(engine);
2189 if (ret)
2190 goto error;
2191
2192 /* We may need to do things with the shrinker which
2193 * require us to immediately switch back to the default
2194 * context. This can cause a problem as pinning the
2195 * default context also requires GTT space which may not
2196 * be available. To avoid this we always pin the default
2197 * context.
2198 */
2199 ret = intel_ring_context_pin(dev_priv->kernel_context, engine);
2200 if (ret)
2201 goto error;
2343 2202
2344 ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE); 2203 ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE);
2345 if (IS_ERR(ringbuf)) { 2204 if (IS_ERR(ringbuf)) {
@@ -2408,6 +2267,10 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
2408 2267
2409 i915_cmd_parser_fini_ring(engine); 2268 i915_cmd_parser_fini_ring(engine);
2410 i915_gem_batch_pool_fini(&engine->batch_pool); 2269 i915_gem_batch_pool_fini(&engine->batch_pool);
2270 intel_engine_fini_breadcrumbs(engine);
2271
2272 intel_ring_context_unpin(dev_priv->kernel_context, engine);
2273
2411 engine->i915 = NULL; 2274 engine->i915 = NULL;
2412} 2275}
2413 2276
@@ -2603,10 +2466,19 @@ void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
2603 memset(engine->semaphore.sync_seqno, 0, 2466 memset(engine->semaphore.sync_seqno, 0,
2604 sizeof(engine->semaphore.sync_seqno)); 2467 sizeof(engine->semaphore.sync_seqno));
2605 2468
2606 engine->set_seqno(engine, seqno); 2469 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
2470 if (engine->irq_seqno_barrier)
2471 engine->irq_seqno_barrier(engine);
2607 engine->last_submitted_seqno = seqno; 2472 engine->last_submitted_seqno = seqno;
2608 2473
2609 engine->hangcheck.seqno = seqno; 2474 engine->hangcheck.seqno = seqno;
2475
2476 /* After manually advancing the seqno, fake the interrupt in case
2477 * there are any waiters for that seqno.
2478 */
2479 rcu_read_lock();
2480 intel_engine_wakeup(engine);
2481 rcu_read_unlock();
2610} 2482}
2611 2483
2612static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine, 2484static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
@@ -2614,32 +2486,38 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
2614{ 2486{
2615 struct drm_i915_private *dev_priv = engine->i915; 2487 struct drm_i915_private *dev_priv = engine->i915;
2616 2488
2489 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
2490
2617 /* Every tail move must follow the sequence below */ 2491 /* Every tail move must follow the sequence below */
2618 2492
2619 /* Disable notification that the ring is IDLE. The GT 2493 /* Disable notification that the ring is IDLE. The GT
2620 * will then assume that it is busy and bring it out of rc6. 2494 * will then assume that it is busy and bring it out of rc6.
2621 */ 2495 */
2622 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 2496 I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
2623 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 2497 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2624 2498
2625 /* Clear the context id. Here be magic! */ 2499 /* Clear the context id. Here be magic! */
2626 I915_WRITE64(GEN6_BSD_RNCID, 0x0); 2500 I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
2627 2501
2628 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 2502 /* Wait for the ring not to be idle, i.e. for it to wake up. */
2629 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) & 2503 if (intel_wait_for_register_fw(dev_priv,
2630 GEN6_BSD_SLEEP_INDICATOR) == 0, 2504 GEN6_BSD_SLEEP_PSMI_CONTROL,
2631 50)) 2505 GEN6_BSD_SLEEP_INDICATOR,
2506 0,
2507 50))
2632 DRM_ERROR("timed out waiting for the BSD ring to wake up\n"); 2508 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2633 2509
2634 /* Now that the ring is fully powered up, update the tail */ 2510 /* Now that the ring is fully powered up, update the tail */
2635 I915_WRITE_TAIL(engine, value); 2511 I915_WRITE_FW(RING_TAIL(engine->mmio_base), value);
2636 POSTING_READ(RING_TAIL(engine->mmio_base)); 2512 POSTING_READ_FW(RING_TAIL(engine->mmio_base));
2637 2513
2638 /* Let the ring send IDLE messages to the GT again, 2514 /* Let the ring send IDLE messages to the GT again,
2639 * and so let it sleep to conserve power when idle. 2515 * and so let it sleep to conserve power when idle.
2640 */ 2516 */
2641 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 2517 I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
2642 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 2518 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2519
2520 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2643} 2521}
2644 2522
2645static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, 2523static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
@@ -2808,11 +2686,159 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
2808 return 0; 2686 return 0;
2809} 2687}
2810 2688
2689static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
2690 struct intel_engine_cs *engine)
2691{
2692 struct drm_i915_gem_object *obj;
2693 int ret, i;
2694
2695 if (!i915_semaphore_is_enabled(dev_priv))
2696 return;
2697
2698 if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore_obj) {
2699 obj = i915_gem_object_create(&dev_priv->drm, 4096);
2700 if (IS_ERR(obj)) {
2701 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2702 i915.semaphores = 0;
2703 } else {
2704 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2705 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2706 if (ret != 0) {
2707 drm_gem_object_unreference(&obj->base);
2708 DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2709 i915.semaphores = 0;
2710 } else {
2711 dev_priv->semaphore_obj = obj;
2712 }
2713 }
2714 }
2715
2716 if (!i915_semaphore_is_enabled(dev_priv))
2717 return;
2718
2719 if (INTEL_GEN(dev_priv) >= 8) {
2720 u64 offset = i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj);
2721
2722 engine->semaphore.sync_to = gen8_ring_sync;
2723 engine->semaphore.signal = gen8_xcs_signal;
2724
2725 for (i = 0; i < I915_NUM_ENGINES; i++) {
2726 u64 ring_offset;
2727
2728 if (i != engine->id)
2729 ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
2730 else
2731 ring_offset = MI_SEMAPHORE_SYNC_INVALID;
2732
2733 engine->semaphore.signal_ggtt[i] = ring_offset;
2734 }
2735 } else if (INTEL_GEN(dev_priv) >= 6) {
2736 engine->semaphore.sync_to = gen6_ring_sync;
2737 engine->semaphore.signal = gen6_signal;
2738
2739 /*
2740 * The current semaphore is only applied on pre-gen8
2741 * platform. And there is no VCS2 ring on the pre-gen8
2742 * platform. So the semaphore between RCS and VCS2 is
2743 * initialized as INVALID. Gen8 will initialize the
2744 * sema between VCS2 and RCS later.
2745 */
2746 for (i = 0; i < I915_NUM_ENGINES; i++) {
2747 static const struct {
2748 u32 wait_mbox;
2749 i915_reg_t mbox_reg;
2750 } sem_data[I915_NUM_ENGINES][I915_NUM_ENGINES] = {
2751 [RCS] = {
2752 [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RV, .mbox_reg = GEN6_VRSYNC },
2753 [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RB, .mbox_reg = GEN6_BRSYNC },
2754 [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
2755 },
2756 [VCS] = {
2757 [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VR, .mbox_reg = GEN6_RVSYNC },
2758 [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VB, .mbox_reg = GEN6_BVSYNC },
2759 [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
2760 },
2761 [BCS] = {
2762 [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BR, .mbox_reg = GEN6_RBSYNC },
2763 [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BV, .mbox_reg = GEN6_VBSYNC },
2764 [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
2765 },
2766 [VECS] = {
2767 [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
2768 [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
2769 [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
2770 },
2771 };
2772 u32 wait_mbox;
2773 i915_reg_t mbox_reg;
2774
2775 if (i == engine->id || i == VCS2) {
2776 wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
2777 mbox_reg = GEN6_NOSYNC;
2778 } else {
2779 wait_mbox = sem_data[engine->id][i].wait_mbox;
2780 mbox_reg = sem_data[engine->id][i].mbox_reg;
2781 }
2782
2783 engine->semaphore.mbox.wait[i] = wait_mbox;
2784 engine->semaphore.mbox.signal[i] = mbox_reg;
2785 }
2786 }
2787}
2788
2789static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
2790 struct intel_engine_cs *engine)
2791{
2792 if (INTEL_GEN(dev_priv) >= 8) {
2793 engine->irq_enable = gen8_irq_enable;
2794 engine->irq_disable = gen8_irq_disable;
2795 engine->irq_seqno_barrier = gen6_seqno_barrier;
2796 } else if (INTEL_GEN(dev_priv) >= 6) {
2797 engine->irq_enable = gen6_irq_enable;
2798 engine->irq_disable = gen6_irq_disable;
2799 engine->irq_seqno_barrier = gen6_seqno_barrier;
2800 } else if (INTEL_GEN(dev_priv) >= 5) {
2801 engine->irq_enable = gen5_irq_enable;
2802 engine->irq_disable = gen5_irq_disable;
2803 engine->irq_seqno_barrier = gen5_seqno_barrier;
2804 } else if (INTEL_GEN(dev_priv) >= 3) {
2805 engine->irq_enable = i9xx_irq_enable;
2806 engine->irq_disable = i9xx_irq_disable;
2807 } else {
2808 engine->irq_enable = i8xx_irq_enable;
2809 engine->irq_disable = i8xx_irq_disable;
2810 }
2811}
2812
2813static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
2814 struct intel_engine_cs *engine)
2815{
2816 engine->init_hw = init_ring_common;
2817 engine->write_tail = ring_write_tail;
2818
2819 engine->add_request = i9xx_add_request;
2820 if (INTEL_GEN(dev_priv) >= 6)
2821 engine->add_request = gen6_add_request;
2822
2823 if (INTEL_GEN(dev_priv) >= 8)
2824 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2825 else if (INTEL_GEN(dev_priv) >= 6)
2826 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2827 else if (INTEL_GEN(dev_priv) >= 4)
2828 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
2829 else if (IS_I830(dev_priv) || IS_845G(dev_priv))
2830 engine->dispatch_execbuffer = i830_dispatch_execbuffer;
2831 else
2832 engine->dispatch_execbuffer = i915_dispatch_execbuffer;
2833
2834 intel_ring_init_irq(dev_priv, engine);
2835 intel_ring_init_semaphores(dev_priv, engine);
2836}
2837
2811int intel_init_render_ring_buffer(struct drm_device *dev) 2838int intel_init_render_ring_buffer(struct drm_device *dev)
2812{ 2839{
2813 struct drm_i915_private *dev_priv = dev->dev_private; 2840 struct drm_i915_private *dev_priv = to_i915(dev);
2814 struct intel_engine_cs *engine = &dev_priv->engine[RCS]; 2841 struct intel_engine_cs *engine = &dev_priv->engine[RCS];
2815 struct drm_i915_gem_object *obj;
2816 int ret; 2842 int ret;
2817 2843
2818 engine->name = "render ring"; 2844 engine->name = "render ring";
@@ -2821,139 +2847,49 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
2821 engine->hw_id = 0; 2847 engine->hw_id = 0;
2822 engine->mmio_base = RENDER_RING_BASE; 2848 engine->mmio_base = RENDER_RING_BASE;
2823 2849
2824 if (INTEL_GEN(dev_priv) >= 8) { 2850 intel_ring_default_vfuncs(dev_priv, engine);
2825 if (i915_semaphore_is_enabled(dev_priv)) { 2851
2826 obj = i915_gem_object_create(dev, 4096); 2852 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2827 if (IS_ERR(obj)) { 2853 if (HAS_L3_DPF(dev_priv))
2828 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n"); 2854 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2829 i915.semaphores = 0;
2830 } else {
2831 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2832 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2833 if (ret != 0) {
2834 drm_gem_object_unreference(&obj->base);
2835 DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2836 i915.semaphores = 0;
2837 } else
2838 dev_priv->semaphore_obj = obj;
2839 }
2840 }
2841 2855
2856 if (INTEL_GEN(dev_priv) >= 8) {
2842 engine->init_context = intel_rcs_ctx_init; 2857 engine->init_context = intel_rcs_ctx_init;
2843 engine->add_request = gen8_render_add_request; 2858 engine->add_request = gen8_render_add_request;
2844 engine->flush = gen8_render_ring_flush; 2859 engine->flush = gen8_render_ring_flush;
2845 engine->irq_get = gen8_ring_get_irq; 2860 if (i915_semaphore_is_enabled(dev_priv))
2846 engine->irq_put = gen8_ring_put_irq;
2847 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2848 engine->get_seqno = ring_get_seqno;
2849 engine->set_seqno = ring_set_seqno;
2850 if (i915_semaphore_is_enabled(dev_priv)) {
2851 WARN_ON(!dev_priv->semaphore_obj);
2852 engine->semaphore.sync_to = gen8_ring_sync;
2853 engine->semaphore.signal = gen8_rcs_signal; 2861 engine->semaphore.signal = gen8_rcs_signal;
2854 GEN8_RING_SEMAPHORE_INIT(engine);
2855 }
2856 } else if (INTEL_GEN(dev_priv) >= 6) { 2862 } else if (INTEL_GEN(dev_priv) >= 6) {
2857 engine->init_context = intel_rcs_ctx_init; 2863 engine->init_context = intel_rcs_ctx_init;
2858 engine->add_request = gen6_add_request;
2859 engine->flush = gen7_render_ring_flush; 2864 engine->flush = gen7_render_ring_flush;
2860 if (IS_GEN6(dev_priv)) 2865 if (IS_GEN6(dev_priv))
2861 engine->flush = gen6_render_ring_flush; 2866 engine->flush = gen6_render_ring_flush;
2862 engine->irq_get = gen6_ring_get_irq;
2863 engine->irq_put = gen6_ring_put_irq;
2864 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2865 engine->irq_seqno_barrier = gen6_seqno_barrier;
2866 engine->get_seqno = ring_get_seqno;
2867 engine->set_seqno = ring_set_seqno;
2868 if (i915_semaphore_is_enabled(dev_priv)) {
2869 engine->semaphore.sync_to = gen6_ring_sync;
2870 engine->semaphore.signal = gen6_signal;
2871 /*
2872 * The current semaphore is only applied on pre-gen8
2873 * platform. And there is no VCS2 ring on the pre-gen8
2874 * platform. So the semaphore between RCS and VCS2 is
2875 * initialized as INVALID. Gen8 will initialize the
2876 * sema between VCS2 and RCS later.
2877 */
2878 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2879 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
2880 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
2881 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
2882 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2883 engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2884 engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
2885 engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
2886 engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
2887 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2888 }
2889 } else if (IS_GEN5(dev_priv)) { 2867 } else if (IS_GEN5(dev_priv)) {
2890 engine->add_request = pc_render_add_request;
2891 engine->flush = gen4_render_ring_flush; 2868 engine->flush = gen4_render_ring_flush;
2892 engine->get_seqno = pc_render_get_seqno;
2893 engine->set_seqno = pc_render_set_seqno;
2894 engine->irq_get = gen5_ring_get_irq;
2895 engine->irq_put = gen5_ring_put_irq;
2896 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
2897 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
2898 } else { 2869 } else {
2899 engine->add_request = i9xx_add_request;
2900 if (INTEL_GEN(dev_priv) < 4) 2870 if (INTEL_GEN(dev_priv) < 4)
2901 engine->flush = gen2_render_ring_flush; 2871 engine->flush = gen2_render_ring_flush;
2902 else 2872 else
2903 engine->flush = gen4_render_ring_flush; 2873 engine->flush = gen4_render_ring_flush;
2904 engine->get_seqno = ring_get_seqno;
2905 engine->set_seqno = ring_set_seqno;
2906 if (IS_GEN2(dev_priv)) {
2907 engine->irq_get = i8xx_ring_get_irq;
2908 engine->irq_put = i8xx_ring_put_irq;
2909 } else {
2910 engine->irq_get = i9xx_ring_get_irq;
2911 engine->irq_put = i9xx_ring_put_irq;
2912 }
2913 engine->irq_enable_mask = I915_USER_INTERRUPT; 2874 engine->irq_enable_mask = I915_USER_INTERRUPT;
2914 } 2875 }
2915 engine->write_tail = ring_write_tail;
2916 2876
2917 if (IS_HASWELL(dev_priv)) 2877 if (IS_HASWELL(dev_priv))
2918 engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; 2878 engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
2919 else if (IS_GEN8(dev_priv)) 2879
2920 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2921 else if (INTEL_GEN(dev_priv) >= 6)
2922 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2923 else if (INTEL_GEN(dev_priv) >= 4)
2924 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
2925 else if (IS_I830(dev_priv) || IS_845G(dev_priv))
2926 engine->dispatch_execbuffer = i830_dispatch_execbuffer;
2927 else
2928 engine->dispatch_execbuffer = i915_dispatch_execbuffer;
2929 engine->init_hw = init_render_ring; 2880 engine->init_hw = init_render_ring;
2930 engine->cleanup = render_ring_cleanup; 2881 engine->cleanup = render_ring_cleanup;
2931 2882
2932 /* Workaround batchbuffer to combat CS tlb bug. */
2933 if (HAS_BROKEN_CS_TLB(dev_priv)) {
2934 obj = i915_gem_object_create(dev, I830_WA_SIZE);
2935 if (IS_ERR(obj)) {
2936 DRM_ERROR("Failed to allocate batch bo\n");
2937 return PTR_ERR(obj);
2938 }
2939
2940 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
2941 if (ret != 0) {
2942 drm_gem_object_unreference(&obj->base);
2943 DRM_ERROR("Failed to ping batch bo\n");
2944 return ret;
2945 }
2946
2947 engine->scratch.obj = obj;
2948 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2949 }
2950
2951 ret = intel_init_ring_buffer(dev, engine); 2883 ret = intel_init_ring_buffer(dev, engine);
2952 if (ret) 2884 if (ret)
2953 return ret; 2885 return ret;
2954 2886
2955 if (INTEL_GEN(dev_priv) >= 5) { 2887 if (INTEL_GEN(dev_priv) >= 6) {
2956 ret = intel_init_pipe_control(engine); 2888 ret = intel_init_pipe_control(engine, 4096);
2889 if (ret)
2890 return ret;
2891 } else if (HAS_BROKEN_CS_TLB(dev_priv)) {
2892 ret = intel_init_pipe_control(engine, I830_WA_SIZE);
2957 if (ret) 2893 if (ret)
2958 return ret; 2894 return ret;
2959 } 2895 }
@@ -2963,7 +2899,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
2963 2899
2964int intel_init_bsd_ring_buffer(struct drm_device *dev) 2900int intel_init_bsd_ring_buffer(struct drm_device *dev)
2965{ 2901{
2966 struct drm_i915_private *dev_priv = dev->dev_private; 2902 struct drm_i915_private *dev_priv = to_i915(dev);
2967 struct intel_engine_cs *engine = &dev_priv->engine[VCS]; 2903 struct intel_engine_cs *engine = &dev_priv->engine[VCS];
2968 2904
2969 engine->name = "bsd ring"; 2905 engine->name = "bsd ring";
@@ -2971,68 +2907,27 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
2971 engine->exec_id = I915_EXEC_BSD; 2907 engine->exec_id = I915_EXEC_BSD;
2972 engine->hw_id = 1; 2908 engine->hw_id = 1;
2973 2909
2974 engine->write_tail = ring_write_tail; 2910 intel_ring_default_vfuncs(dev_priv, engine);
2911
2975 if (INTEL_GEN(dev_priv) >= 6) { 2912 if (INTEL_GEN(dev_priv) >= 6) {
2976 engine->mmio_base = GEN6_BSD_RING_BASE; 2913 engine->mmio_base = GEN6_BSD_RING_BASE;
2977 /* gen6 bsd needs a special wa for tail updates */ 2914 /* gen6 bsd needs a special wa for tail updates */
2978 if (IS_GEN6(dev_priv)) 2915 if (IS_GEN6(dev_priv))
2979 engine->write_tail = gen6_bsd_ring_write_tail; 2916 engine->write_tail = gen6_bsd_ring_write_tail;
2980 engine->flush = gen6_bsd_ring_flush; 2917 engine->flush = gen6_bsd_ring_flush;
2981 engine->add_request = gen6_add_request; 2918 if (INTEL_GEN(dev_priv) >= 8)
2982 engine->irq_seqno_barrier = gen6_seqno_barrier;
2983 engine->get_seqno = ring_get_seqno;
2984 engine->set_seqno = ring_set_seqno;
2985 if (INTEL_GEN(dev_priv) >= 8) {
2986 engine->irq_enable_mask = 2919 engine->irq_enable_mask =
2987 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; 2920 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2988 engine->irq_get = gen8_ring_get_irq; 2921 else
2989 engine->irq_put = gen8_ring_put_irq;
2990 engine->dispatch_execbuffer =
2991 gen8_ring_dispatch_execbuffer;
2992 if (i915_semaphore_is_enabled(dev_priv)) {
2993 engine->semaphore.sync_to = gen8_ring_sync;
2994 engine->semaphore.signal = gen8_xcs_signal;
2995 GEN8_RING_SEMAPHORE_INIT(engine);
2996 }
2997 } else {
2998 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 2922 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2999 engine->irq_get = gen6_ring_get_irq;
3000 engine->irq_put = gen6_ring_put_irq;
3001 engine->dispatch_execbuffer =
3002 gen6_ring_dispatch_execbuffer;
3003 if (i915_semaphore_is_enabled(dev_priv)) {
3004 engine->semaphore.sync_to = gen6_ring_sync;
3005 engine->semaphore.signal = gen6_signal;
3006 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
3007 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
3008 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
3009 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
3010 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3011 engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
3012 engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
3013 engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
3014 engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
3015 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3016 }
3017 }
3018 } else { 2923 } else {
3019 engine->mmio_base = BSD_RING_BASE; 2924 engine->mmio_base = BSD_RING_BASE;
3020 engine->flush = bsd_ring_flush; 2925 engine->flush = bsd_ring_flush;
3021 engine->add_request = i9xx_add_request; 2926 if (IS_GEN5(dev_priv))
3022 engine->get_seqno = ring_get_seqno;
3023 engine->set_seqno = ring_set_seqno;
3024 if (IS_GEN5(dev_priv)) {
3025 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 2927 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3026 engine->irq_get = gen5_ring_get_irq; 2928 else
3027 engine->irq_put = gen5_ring_put_irq;
3028 } else {
3029 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 2929 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
3030 engine->irq_get = i9xx_ring_get_irq;
3031 engine->irq_put = i9xx_ring_put_irq;
3032 }
3033 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
3034 } 2930 }
3035 engine->init_hw = init_ring_common;
3036 2931
3037 return intel_init_ring_buffer(dev, engine); 2932 return intel_init_ring_buffer(dev, engine);
3038} 2933}
@@ -3042,147 +2937,70 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
3042 */ 2937 */
3043int intel_init_bsd2_ring_buffer(struct drm_device *dev) 2938int intel_init_bsd2_ring_buffer(struct drm_device *dev)
3044{ 2939{
3045 struct drm_i915_private *dev_priv = dev->dev_private; 2940 struct drm_i915_private *dev_priv = to_i915(dev);
3046 struct intel_engine_cs *engine = &dev_priv->engine[VCS2]; 2941 struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
3047 2942
3048 engine->name = "bsd2 ring"; 2943 engine->name = "bsd2 ring";
3049 engine->id = VCS2; 2944 engine->id = VCS2;
3050 engine->exec_id = I915_EXEC_BSD; 2945 engine->exec_id = I915_EXEC_BSD;
3051 engine->hw_id = 4; 2946 engine->hw_id = 4;
3052
3053 engine->write_tail = ring_write_tail;
3054 engine->mmio_base = GEN8_BSD2_RING_BASE; 2947 engine->mmio_base = GEN8_BSD2_RING_BASE;
2948
2949 intel_ring_default_vfuncs(dev_priv, engine);
2950
3055 engine->flush = gen6_bsd_ring_flush; 2951 engine->flush = gen6_bsd_ring_flush;
3056 engine->add_request = gen6_add_request;
3057 engine->irq_seqno_barrier = gen6_seqno_barrier;
3058 engine->get_seqno = ring_get_seqno;
3059 engine->set_seqno = ring_set_seqno;
3060 engine->irq_enable_mask = 2952 engine->irq_enable_mask =
3061 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT; 2953 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
3062 engine->irq_get = gen8_ring_get_irq;
3063 engine->irq_put = gen8_ring_put_irq;
3064 engine->dispatch_execbuffer =
3065 gen8_ring_dispatch_execbuffer;
3066 if (i915_semaphore_is_enabled(dev_priv)) {
3067 engine->semaphore.sync_to = gen8_ring_sync;
3068 engine->semaphore.signal = gen8_xcs_signal;
3069 GEN8_RING_SEMAPHORE_INIT(engine);
3070 }
3071 engine->init_hw = init_ring_common;
3072 2954
3073 return intel_init_ring_buffer(dev, engine); 2955 return intel_init_ring_buffer(dev, engine);
3074} 2956}
3075 2957
3076int intel_init_blt_ring_buffer(struct drm_device *dev) 2958int intel_init_blt_ring_buffer(struct drm_device *dev)
3077{ 2959{
3078 struct drm_i915_private *dev_priv = dev->dev_private; 2960 struct drm_i915_private *dev_priv = to_i915(dev);
3079 struct intel_engine_cs *engine = &dev_priv->engine[BCS]; 2961 struct intel_engine_cs *engine = &dev_priv->engine[BCS];
3080 2962
3081 engine->name = "blitter ring"; 2963 engine->name = "blitter ring";
3082 engine->id = BCS; 2964 engine->id = BCS;
3083 engine->exec_id = I915_EXEC_BLT; 2965 engine->exec_id = I915_EXEC_BLT;
3084 engine->hw_id = 2; 2966 engine->hw_id = 2;
3085
3086 engine->mmio_base = BLT_RING_BASE; 2967 engine->mmio_base = BLT_RING_BASE;
3087 engine->write_tail = ring_write_tail; 2968
2969 intel_ring_default_vfuncs(dev_priv, engine);
2970
3088 engine->flush = gen6_ring_flush; 2971 engine->flush = gen6_ring_flush;
3089 engine->add_request = gen6_add_request; 2972 if (INTEL_GEN(dev_priv) >= 8)
3090 engine->irq_seqno_barrier = gen6_seqno_barrier;
3091 engine->get_seqno = ring_get_seqno;
3092 engine->set_seqno = ring_set_seqno;
3093 if (INTEL_GEN(dev_priv) >= 8) {
3094 engine->irq_enable_mask = 2973 engine->irq_enable_mask =
3095 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; 2974 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
3096 engine->irq_get = gen8_ring_get_irq; 2975 else
3097 engine->irq_put = gen8_ring_put_irq;
3098 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3099 if (i915_semaphore_is_enabled(dev_priv)) {
3100 engine->semaphore.sync_to = gen8_ring_sync;
3101 engine->semaphore.signal = gen8_xcs_signal;
3102 GEN8_RING_SEMAPHORE_INIT(engine);
3103 }
3104 } else {
3105 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 2976 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
3106 engine->irq_get = gen6_ring_get_irq;
3107 engine->irq_put = gen6_ring_put_irq;
3108 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3109 if (i915_semaphore_is_enabled(dev_priv)) {
3110 engine->semaphore.signal = gen6_signal;
3111 engine->semaphore.sync_to = gen6_ring_sync;
3112 /*
3113 * The current semaphore is only applied on pre-gen8
3114 * platform. And there is no VCS2 ring on the pre-gen8
3115 * platform. So the semaphore between BCS and VCS2 is
3116 * initialized as INVALID. Gen8 will initialize the
3117 * sema between BCS and VCS2 later.
3118 */
3119 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
3120 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
3121 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
3122 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
3123 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3124 engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
3125 engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
3126 engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
3127 engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
3128 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3129 }
3130 }
3131 engine->init_hw = init_ring_common;
3132 2977
3133 return intel_init_ring_buffer(dev, engine); 2978 return intel_init_ring_buffer(dev, engine);
3134} 2979}
3135 2980
3136int intel_init_vebox_ring_buffer(struct drm_device *dev) 2981int intel_init_vebox_ring_buffer(struct drm_device *dev)
3137{ 2982{
3138 struct drm_i915_private *dev_priv = dev->dev_private; 2983 struct drm_i915_private *dev_priv = to_i915(dev);
3139 struct intel_engine_cs *engine = &dev_priv->engine[VECS]; 2984 struct intel_engine_cs *engine = &dev_priv->engine[VECS];
3140 2985
3141 engine->name = "video enhancement ring"; 2986 engine->name = "video enhancement ring";
3142 engine->id = VECS; 2987 engine->id = VECS;
3143 engine->exec_id = I915_EXEC_VEBOX; 2988 engine->exec_id = I915_EXEC_VEBOX;
3144 engine->hw_id = 3; 2989 engine->hw_id = 3;
3145
3146 engine->mmio_base = VEBOX_RING_BASE; 2990 engine->mmio_base = VEBOX_RING_BASE;
3147 engine->write_tail = ring_write_tail; 2991
2992 intel_ring_default_vfuncs(dev_priv, engine);
2993
3148 engine->flush = gen6_ring_flush; 2994 engine->flush = gen6_ring_flush;
3149 engine->add_request = gen6_add_request;
3150 engine->irq_seqno_barrier = gen6_seqno_barrier;
3151 engine->get_seqno = ring_get_seqno;
3152 engine->set_seqno = ring_set_seqno;
3153 2995
3154 if (INTEL_GEN(dev_priv) >= 8) { 2996 if (INTEL_GEN(dev_priv) >= 8) {
3155 engine->irq_enable_mask = 2997 engine->irq_enable_mask =
3156 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT; 2998 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
3157 engine->irq_get = gen8_ring_get_irq;
3158 engine->irq_put = gen8_ring_put_irq;
3159 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3160 if (i915_semaphore_is_enabled(dev_priv)) {
3161 engine->semaphore.sync_to = gen8_ring_sync;
3162 engine->semaphore.signal = gen8_xcs_signal;
3163 GEN8_RING_SEMAPHORE_INIT(engine);
3164 }
3165 } else { 2999 } else {
3166 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 3000 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
3167 engine->irq_get = hsw_vebox_get_irq; 3001 engine->irq_enable = hsw_vebox_irq_enable;
3168 engine->irq_put = hsw_vebox_put_irq; 3002 engine->irq_disable = hsw_vebox_irq_disable;
3169 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3170 if (i915_semaphore_is_enabled(dev_priv)) {
3171 engine->semaphore.sync_to = gen6_ring_sync;
3172 engine->semaphore.signal = gen6_signal;
3173 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
3174 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
3175 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
3176 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
3177 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3178 engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
3179 engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
3180 engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
3181 engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
3182 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3183 }
3184 } 3003 }
3185 engine->init_hw = init_ring_common;
3186 3004
3187 return intel_init_ring_buffer(dev, engine); 3005 return intel_init_ring_buffer(dev, engine);
3188} 3006}
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-03 06:21:19 -0400