diff options
Diffstat (limited to 'drivers/gpu/drm/i915/intel_ringbuffer.c')
-rw-r--r-- | drivers/gpu/drm/i915/intel_ringbuffer.c | 420 |
1 files changed, 285 insertions, 135 deletions
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 2faef2605e97..599709e80a16 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c | |||
@@ -48,9 +48,8 @@ static inline int __ring_space(int head, int tail, int size) | |||
48 | return space; | 48 | return space; |
49 | } | 49 | } |
50 | 50 | ||
51 | static inline int ring_space(struct intel_engine_cs *ring) | 51 | static inline int ring_space(struct intel_ringbuffer *ringbuf) |
52 | { | 52 | { |
53 | struct intel_ringbuffer *ringbuf = ring->buffer; | ||
54 | return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size); | 53 | return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size); |
55 | } | 54 | } |
56 | 55 | ||
@@ -545,7 +544,7 @@ static int init_ring_common(struct intel_engine_cs *ring) | |||
545 | else { | 544 | else { |
546 | ringbuf->head = I915_READ_HEAD(ring); | 545 | ringbuf->head = I915_READ_HEAD(ring); |
547 | ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; | 546 | ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; |
548 | ringbuf->space = ring_space(ring); | 547 | ringbuf->space = ring_space(ringbuf); |
549 | ringbuf->last_retired_head = -1; | 548 | ringbuf->last_retired_head = -1; |
550 | } | 549 | } |
551 | 550 | ||
@@ -660,6 +659,13 @@ static int init_render_ring(struct intel_engine_cs *ring) | |||
660 | static void render_ring_cleanup(struct intel_engine_cs *ring) | 659 | static void render_ring_cleanup(struct intel_engine_cs *ring) |
661 | { | 660 | { |
662 | struct drm_device *dev = ring->dev; | 661 | struct drm_device *dev = ring->dev; |
662 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
663 | |||
664 | if (dev_priv->semaphore_obj) { | ||
665 | i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj); | ||
666 | drm_gem_object_unreference(&dev_priv->semaphore_obj->base); | ||
667 | dev_priv->semaphore_obj = NULL; | ||
668 | } | ||
663 | 669 | ||
664 | if (ring->scratch.obj == NULL) | 670 | if (ring->scratch.obj == NULL) |
665 | return; | 671 | return; |
@@ -673,29 +679,96 @@ static void render_ring_cleanup(struct intel_engine_cs *ring) | |||
673 | ring->scratch.obj = NULL; | 679 | ring->scratch.obj = NULL; |
674 | } | 680 | } |
675 | 681 | ||
682 | static int gen8_rcs_signal(struct intel_engine_cs *signaller, | ||
683 | unsigned int num_dwords) | ||
684 | { | ||
685 | #define MBOX_UPDATE_DWORDS 8 | ||
686 | struct drm_device *dev = signaller->dev; | ||
687 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
688 | struct intel_engine_cs *waiter; | ||
689 | int i, ret, num_rings; | ||
690 | |||
691 | num_rings = hweight32(INTEL_INFO(dev)->ring_mask); | ||
692 | num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; | ||
693 | #undef MBOX_UPDATE_DWORDS | ||
694 | |||
695 | ret = intel_ring_begin(signaller, num_dwords); | ||
696 | if (ret) | ||
697 | return ret; | ||
698 | |||
699 | for_each_ring(waiter, dev_priv, i) { | ||
700 | u64 gtt_offset = signaller->semaphore.signal_ggtt[i]; | ||
701 | if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) | ||
702 | continue; | ||
703 | |||
704 | intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6)); | ||
705 | intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB | | ||
706 | PIPE_CONTROL_QW_WRITE | | ||
707 | PIPE_CONTROL_FLUSH_ENABLE); | ||
708 | intel_ring_emit(signaller, lower_32_bits(gtt_offset)); | ||
709 | intel_ring_emit(signaller, upper_32_bits(gtt_offset)); | ||
710 | intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); | ||
711 | intel_ring_emit(signaller, 0); | ||
712 | intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | | ||
713 | MI_SEMAPHORE_TARGET(waiter->id)); | ||
714 | intel_ring_emit(signaller, 0); | ||
715 | } | ||
716 | |||
717 | return 0; | ||
718 | } | ||
719 | |||
720 | static int gen8_xcs_signal(struct intel_engine_cs *signaller, | ||
721 | unsigned int num_dwords) | ||
722 | { | ||
723 | #define MBOX_UPDATE_DWORDS 6 | ||
724 | struct drm_device *dev = signaller->dev; | ||
725 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
726 | struct intel_engine_cs *waiter; | ||
727 | int i, ret, num_rings; | ||
728 | |||
729 | num_rings = hweight32(INTEL_INFO(dev)->ring_mask); | ||
730 | num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; | ||
731 | #undef MBOX_UPDATE_DWORDS | ||
732 | |||
733 | ret = intel_ring_begin(signaller, num_dwords); | ||
734 | if (ret) | ||
735 | return ret; | ||
736 | |||
737 | for_each_ring(waiter, dev_priv, i) { | ||
738 | u64 gtt_offset = signaller->semaphore.signal_ggtt[i]; | ||
739 | if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) | ||
740 | continue; | ||
741 | |||
742 | intel_ring_emit(signaller, (MI_FLUSH_DW + 1) | | ||
743 | MI_FLUSH_DW_OP_STOREDW); | ||
744 | intel_ring_emit(signaller, lower_32_bits(gtt_offset) | | ||
745 | MI_FLUSH_DW_USE_GTT); | ||
746 | intel_ring_emit(signaller, upper_32_bits(gtt_offset)); | ||
747 | intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); | ||
748 | intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | | ||
749 | MI_SEMAPHORE_TARGET(waiter->id)); | ||
750 | intel_ring_emit(signaller, 0); | ||
751 | } | ||
752 | |||
753 | return 0; | ||
754 | } | ||
755 | |||
676 | static int gen6_signal(struct intel_engine_cs *signaller, | 756 | static int gen6_signal(struct intel_engine_cs *signaller, |
677 | unsigned int num_dwords) | 757 | unsigned int num_dwords) |
678 | { | 758 | { |
679 | struct drm_device *dev = signaller->dev; | 759 | struct drm_device *dev = signaller->dev; |
680 | struct drm_i915_private *dev_priv = dev->dev_private; | 760 | struct drm_i915_private *dev_priv = dev->dev_private; |
681 | struct intel_engine_cs *useless; | 761 | struct intel_engine_cs *useless; |
682 | int i, ret; | 762 | int i, ret, num_rings; |
683 | 763 | ||
684 | /* NB: In order to be able to do semaphore MBOX updates for varying | 764 | #define MBOX_UPDATE_DWORDS 3 |
685 | * number of rings, it's easiest if we round up each individual update | 765 | num_rings = hweight32(INTEL_INFO(dev)->ring_mask); |
686 | * to a multiple of 2 (since ring updates must always be a multiple of | 766 | num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2); |
687 | * 2) even though the actual update only requires 3 dwords. | 767 | #undef MBOX_UPDATE_DWORDS |
688 | */ | ||
689 | #define MBOX_UPDATE_DWORDS 4 | ||
690 | if (i915_semaphore_is_enabled(dev)) | ||
691 | num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS); | ||
692 | else | ||
693 | return intel_ring_begin(signaller, num_dwords); | ||
694 | 768 | ||
695 | ret = intel_ring_begin(signaller, num_dwords); | 769 | ret = intel_ring_begin(signaller, num_dwords); |
696 | if (ret) | 770 | if (ret) |
697 | return ret; | 771 | return ret; |
698 | #undef MBOX_UPDATE_DWORDS | ||
699 | 772 | ||
700 | for_each_ring(useless, dev_priv, i) { | 773 | for_each_ring(useless, dev_priv, i) { |
701 | u32 mbox_reg = signaller->semaphore.mbox.signal[i]; | 774 | u32 mbox_reg = signaller->semaphore.mbox.signal[i]; |
@@ -703,15 +776,13 @@ static int gen6_signal(struct intel_engine_cs *signaller, | |||
703 | intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); | 776 | intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); |
704 | intel_ring_emit(signaller, mbox_reg); | 777 | intel_ring_emit(signaller, mbox_reg); |
705 | intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); | 778 | intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); |
706 | intel_ring_emit(signaller, MI_NOOP); | ||
707 | } else { | ||
708 | intel_ring_emit(signaller, MI_NOOP); | ||
709 | intel_ring_emit(signaller, MI_NOOP); | ||
710 | intel_ring_emit(signaller, MI_NOOP); | ||
711 | intel_ring_emit(signaller, MI_NOOP); | ||
712 | } | 779 | } |
713 | } | 780 | } |
714 | 781 | ||
782 | /* If num_dwords was rounded, make sure the tail pointer is correct */ | ||
783 | if (num_rings % 2 == 0) | ||
784 | intel_ring_emit(signaller, MI_NOOP); | ||
785 | |||
715 | return 0; | 786 | return 0; |
716 | } | 787 | } |
717 | 788 | ||
@@ -729,7 +800,11 @@ gen6_add_request(struct intel_engine_cs *ring) | |||
729 | { | 800 | { |
730 | int ret; | 801 | int ret; |
731 | 802 | ||
732 | ret = ring->semaphore.signal(ring, 4); | 803 | if (ring->semaphore.signal) |
804 | ret = ring->semaphore.signal(ring, 4); | ||
805 | else | ||
806 | ret = intel_ring_begin(ring, 4); | ||
807 | |||
733 | if (ret) | 808 | if (ret) |
734 | return ret; | 809 | return ret; |
735 | 810 | ||
@@ -756,6 +831,32 @@ static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, | |||
756 | * @signaller - ring which has, or will signal | 831 | * @signaller - ring which has, or will signal |
757 | * @seqno - seqno which the waiter will block on | 832 | * @seqno - seqno which the waiter will block on |
758 | */ | 833 | */ |
834 | |||
835 | static int | ||
836 | gen8_ring_sync(struct intel_engine_cs *waiter, | ||
837 | struct intel_engine_cs *signaller, | ||
838 | u32 seqno) | ||
839 | { | ||
840 | struct drm_i915_private *dev_priv = waiter->dev->dev_private; | ||
841 | int ret; | ||
842 | |||
843 | ret = intel_ring_begin(waiter, 4); | ||
844 | if (ret) | ||
845 | return ret; | ||
846 | |||
847 | intel_ring_emit(waiter, MI_SEMAPHORE_WAIT | | ||
848 | MI_SEMAPHORE_GLOBAL_GTT | | ||
849 | MI_SEMAPHORE_POLL | | ||
850 | MI_SEMAPHORE_SAD_GTE_SDD); | ||
851 | intel_ring_emit(waiter, seqno); | ||
852 | intel_ring_emit(waiter, | ||
853 | lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); | ||
854 | intel_ring_emit(waiter, | ||
855 | upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); | ||
856 | intel_ring_advance(waiter); | ||
857 | return 0; | ||
858 | } | ||
859 | |||
759 | static int | 860 | static int |
760 | gen6_ring_sync(struct intel_engine_cs *waiter, | 861 | gen6_ring_sync(struct intel_engine_cs *waiter, |
761 | struct intel_engine_cs *signaller, | 862 | struct intel_engine_cs *signaller, |
@@ -1331,6 +1432,7 @@ static int init_status_page(struct intel_engine_cs *ring) | |||
1331 | struct drm_i915_gem_object *obj; | 1432 | struct drm_i915_gem_object *obj; |
1332 | 1433 | ||
1333 | if ((obj = ring->status_page.obj) == NULL) { | 1434 | if ((obj = ring->status_page.obj) == NULL) { |
1435 | unsigned flags; | ||
1334 | int ret; | 1436 | int ret; |
1335 | 1437 | ||
1336 | obj = i915_gem_alloc_object(ring->dev, 4096); | 1438 | obj = i915_gem_alloc_object(ring->dev, 4096); |
@@ -1343,7 +1445,20 @@ static int init_status_page(struct intel_engine_cs *ring) | |||
1343 | if (ret) | 1445 | if (ret) |
1344 | goto err_unref; | 1446 | goto err_unref; |
1345 | 1447 | ||
1346 | ret = i915_gem_obj_ggtt_pin(obj, 4096, 0); | 1448 | flags = 0; |
1449 | if (!HAS_LLC(ring->dev)) | ||
1450 | /* On g33, we cannot place HWS above 256MiB, so | ||
1451 | * restrict its pinning to the low mappable arena. | ||
1452 | * Though this restriction is not documented for | ||
1453 | * gen4, gen5, or byt, they also behave similarly | ||
1454 | * and hang if the HWS is placed at the top of the | ||
1455 | * GTT. To generalise, it appears that all !llc | ||
1456 | * platforms have issues with us placing the HWS | ||
1457 | * above the mappable region (even though we never | ||
1458 | * actualy map it). | ||
1459 | */ | ||
1460 | flags |= PIN_MAPPABLE; | ||
1461 | ret = i915_gem_obj_ggtt_pin(obj, 4096, flags); | ||
1347 | if (ret) { | 1462 | if (ret) { |
1348 | err_unref: | 1463 | err_unref: |
1349 | drm_gem_object_unreference(&obj->base); | 1464 | drm_gem_object_unreference(&obj->base); |
@@ -1380,15 +1495,25 @@ static int init_phys_status_page(struct intel_engine_cs *ring) | |||
1380 | return 0; | 1495 | return 0; |
1381 | } | 1496 | } |
1382 | 1497 | ||
1383 | static int allocate_ring_buffer(struct intel_engine_cs *ring) | 1498 | static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf) |
1499 | { | ||
1500 | if (!ringbuf->obj) | ||
1501 | return; | ||
1502 | |||
1503 | iounmap(ringbuf->virtual_start); | ||
1504 | i915_gem_object_ggtt_unpin(ringbuf->obj); | ||
1505 | drm_gem_object_unreference(&ringbuf->obj->base); | ||
1506 | ringbuf->obj = NULL; | ||
1507 | } | ||
1508 | |||
1509 | static int intel_alloc_ringbuffer_obj(struct drm_device *dev, | ||
1510 | struct intel_ringbuffer *ringbuf) | ||
1384 | { | 1511 | { |
1385 | struct drm_device *dev = ring->dev; | ||
1386 | struct drm_i915_private *dev_priv = to_i915(dev); | 1512 | struct drm_i915_private *dev_priv = to_i915(dev); |
1387 | struct intel_ringbuffer *ringbuf = ring->buffer; | ||
1388 | struct drm_i915_gem_object *obj; | 1513 | struct drm_i915_gem_object *obj; |
1389 | int ret; | 1514 | int ret; |
1390 | 1515 | ||
1391 | if (intel_ring_initialized(ring)) | 1516 | if (ringbuf->obj) |
1392 | return 0; | 1517 | return 0; |
1393 | 1518 | ||
1394 | obj = NULL; | 1519 | obj = NULL; |
@@ -1460,7 +1585,7 @@ static int intel_init_ring_buffer(struct drm_device *dev, | |||
1460 | goto error; | 1585 | goto error; |
1461 | } | 1586 | } |
1462 | 1587 | ||
1463 | ret = allocate_ring_buffer(ring); | 1588 | ret = intel_alloc_ringbuffer_obj(dev, ringbuf); |
1464 | if (ret) { | 1589 | if (ret) { |
1465 | DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret); | 1590 | DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret); |
1466 | goto error; | 1591 | goto error; |
@@ -1501,11 +1626,7 @@ void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) | |||
1501 | intel_stop_ring_buffer(ring); | 1626 | intel_stop_ring_buffer(ring); |
1502 | WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); | 1627 | WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); |
1503 | 1628 | ||
1504 | iounmap(ringbuf->virtual_start); | 1629 | intel_destroy_ringbuffer_obj(ringbuf); |
1505 | |||
1506 | i915_gem_object_ggtt_unpin(ringbuf->obj); | ||
1507 | drm_gem_object_unreference(&ringbuf->obj->base); | ||
1508 | ringbuf->obj = NULL; | ||
1509 | ring->preallocated_lazy_request = NULL; | 1630 | ring->preallocated_lazy_request = NULL; |
1510 | ring->outstanding_lazy_seqno = 0; | 1631 | ring->outstanding_lazy_seqno = 0; |
1511 | 1632 | ||
@@ -1531,7 +1652,7 @@ static int intel_ring_wait_request(struct intel_engine_cs *ring, int n) | |||
1531 | ringbuf->head = ringbuf->last_retired_head; | 1652 | ringbuf->head = ringbuf->last_retired_head; |
1532 | ringbuf->last_retired_head = -1; | 1653 | ringbuf->last_retired_head = -1; |
1533 | 1654 | ||
1534 | ringbuf->space = ring_space(ring); | 1655 | ringbuf->space = ring_space(ringbuf); |
1535 | if (ringbuf->space >= n) | 1656 | if (ringbuf->space >= n) |
1536 | return 0; | 1657 | return 0; |
1537 | } | 1658 | } |
@@ -1554,7 +1675,7 @@ static int intel_ring_wait_request(struct intel_engine_cs *ring, int n) | |||
1554 | ringbuf->head = ringbuf->last_retired_head; | 1675 | ringbuf->head = ringbuf->last_retired_head; |
1555 | ringbuf->last_retired_head = -1; | 1676 | ringbuf->last_retired_head = -1; |
1556 | 1677 | ||
1557 | ringbuf->space = ring_space(ring); | 1678 | ringbuf->space = ring_space(ringbuf); |
1558 | return 0; | 1679 | return 0; |
1559 | } | 1680 | } |
1560 | 1681 | ||
@@ -1583,7 +1704,7 @@ static int ring_wait_for_space(struct intel_engine_cs *ring, int n) | |||
1583 | trace_i915_ring_wait_begin(ring); | 1704 | trace_i915_ring_wait_begin(ring); |
1584 | do { | 1705 | do { |
1585 | ringbuf->head = I915_READ_HEAD(ring); | 1706 | ringbuf->head = I915_READ_HEAD(ring); |
1586 | ringbuf->space = ring_space(ring); | 1707 | ringbuf->space = ring_space(ringbuf); |
1587 | if (ringbuf->space >= n) { | 1708 | if (ringbuf->space >= n) { |
1588 | ret = 0; | 1709 | ret = 0; |
1589 | break; | 1710 | break; |
@@ -1635,7 +1756,7 @@ static int intel_wrap_ring_buffer(struct intel_engine_cs *ring) | |||
1635 | iowrite32(MI_NOOP, virt++); | 1756 | iowrite32(MI_NOOP, virt++); |
1636 | 1757 | ||
1637 | ringbuf->tail = 0; | 1758 | ringbuf->tail = 0; |
1638 | ringbuf->space = ring_space(ring); | 1759 | ringbuf->space = ring_space(ringbuf); |
1639 | 1760 | ||
1640 | return 0; | 1761 | return 0; |
1641 | } | 1762 | } |
@@ -1947,45 +2068,74 @@ int intel_init_render_ring_buffer(struct drm_device *dev) | |||
1947 | { | 2068 | { |
1948 | struct drm_i915_private *dev_priv = dev->dev_private; | 2069 | struct drm_i915_private *dev_priv = dev->dev_private; |
1949 | struct intel_engine_cs *ring = &dev_priv->ring[RCS]; | 2070 | struct intel_engine_cs *ring = &dev_priv->ring[RCS]; |
2071 | struct drm_i915_gem_object *obj; | ||
2072 | int ret; | ||
1950 | 2073 | ||
1951 | ring->name = "render ring"; | 2074 | ring->name = "render ring"; |
1952 | ring->id = RCS; | 2075 | ring->id = RCS; |
1953 | ring->mmio_base = RENDER_RING_BASE; | 2076 | ring->mmio_base = RENDER_RING_BASE; |
1954 | 2077 | ||
1955 | if (INTEL_INFO(dev)->gen >= 6) { | 2078 | if (INTEL_INFO(dev)->gen >= 8) { |
2079 | if (i915_semaphore_is_enabled(dev)) { | ||
2080 | obj = i915_gem_alloc_object(dev, 4096); | ||
2081 | if (obj == NULL) { | ||
2082 | DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n"); | ||
2083 | i915.semaphores = 0; | ||
2084 | } else { | ||
2085 | i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); | ||
2086 | ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK); | ||
2087 | if (ret != 0) { | ||
2088 | drm_gem_object_unreference(&obj->base); | ||
2089 | DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n"); | ||
2090 | i915.semaphores = 0; | ||
2091 | } else | ||
2092 | dev_priv->semaphore_obj = obj; | ||
2093 | } | ||
2094 | } | ||
2095 | ring->add_request = gen6_add_request; | ||
2096 | ring->flush = gen8_render_ring_flush; | ||
2097 | ring->irq_get = gen8_ring_get_irq; | ||
2098 | ring->irq_put = gen8_ring_put_irq; | ||
2099 | ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; | ||
2100 | ring->get_seqno = gen6_ring_get_seqno; | ||
2101 | ring->set_seqno = ring_set_seqno; | ||
2102 | if (i915_semaphore_is_enabled(dev)) { | ||
2103 | WARN_ON(!dev_priv->semaphore_obj); | ||
2104 | ring->semaphore.sync_to = gen8_ring_sync; | ||
2105 | ring->semaphore.signal = gen8_rcs_signal; | ||
2106 | GEN8_RING_SEMAPHORE_INIT; | ||
2107 | } | ||
2108 | } else if (INTEL_INFO(dev)->gen >= 6) { | ||
1956 | ring->add_request = gen6_add_request; | 2109 | ring->add_request = gen6_add_request; |
1957 | ring->flush = gen7_render_ring_flush; | 2110 | ring->flush = gen7_render_ring_flush; |
1958 | if (INTEL_INFO(dev)->gen == 6) | 2111 | if (INTEL_INFO(dev)->gen == 6) |
1959 | ring->flush = gen6_render_ring_flush; | 2112 | ring->flush = gen6_render_ring_flush; |
1960 | if (INTEL_INFO(dev)->gen >= 8) { | 2113 | ring->irq_get = gen6_ring_get_irq; |
1961 | ring->flush = gen8_render_ring_flush; | 2114 | ring->irq_put = gen6_ring_put_irq; |
1962 | ring->irq_get = gen8_ring_get_irq; | ||
1963 | ring->irq_put = gen8_ring_put_irq; | ||
1964 | } else { | ||
1965 | ring->irq_get = gen6_ring_get_irq; | ||
1966 | ring->irq_put = gen6_ring_put_irq; | ||
1967 | } | ||
1968 | ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; | 2115 | ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; |
1969 | ring->get_seqno = gen6_ring_get_seqno; | 2116 | ring->get_seqno = gen6_ring_get_seqno; |
1970 | ring->set_seqno = ring_set_seqno; | 2117 | ring->set_seqno = ring_set_seqno; |
1971 | ring->semaphore.sync_to = gen6_ring_sync; | 2118 | if (i915_semaphore_is_enabled(dev)) { |
1972 | ring->semaphore.signal = gen6_signal; | 2119 | ring->semaphore.sync_to = gen6_ring_sync; |
1973 | /* | 2120 | ring->semaphore.signal = gen6_signal; |
1974 | * The current semaphore is only applied on pre-gen8 platform. | 2121 | /* |
1975 | * And there is no VCS2 ring on the pre-gen8 platform. So the | 2122 | * The current semaphore is only applied on pre-gen8 |
1976 | * semaphore between RCS and VCS2 is initialized as INVALID. | 2123 | * platform. And there is no VCS2 ring on the pre-gen8 |
1977 | * Gen8 will initialize the sema between VCS2 and RCS later. | 2124 | * platform. So the semaphore between RCS and VCS2 is |
1978 | */ | 2125 | * initialized as INVALID. Gen8 will initialize the |
1979 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; | 2126 | * sema between VCS2 and RCS later. |
1980 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; | 2127 | */ |
1981 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; | 2128 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; |
1982 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; | 2129 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; |
1983 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | 2130 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; |
1984 | ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; | 2131 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; |
1985 | ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; | 2132 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; |
1986 | ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; | 2133 | ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; |
1987 | ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; | 2134 | ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; |
1988 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | 2135 | ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; |
2136 | ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; | ||
2137 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2138 | } | ||
1989 | } else if (IS_GEN5(dev)) { | 2139 | } else if (IS_GEN5(dev)) { |
1990 | ring->add_request = pc_render_add_request; | 2140 | ring->add_request = pc_render_add_request; |
1991 | ring->flush = gen4_render_ring_flush; | 2141 | ring->flush = gen4_render_ring_flush; |
@@ -2013,6 +2163,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev) | |||
2013 | ring->irq_enable_mask = I915_USER_INTERRUPT; | 2163 | ring->irq_enable_mask = I915_USER_INTERRUPT; |
2014 | } | 2164 | } |
2015 | ring->write_tail = ring_write_tail; | 2165 | ring->write_tail = ring_write_tail; |
2166 | |||
2016 | if (IS_HASWELL(dev)) | 2167 | if (IS_HASWELL(dev)) |
2017 | ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; | 2168 | ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; |
2018 | else if (IS_GEN8(dev)) | 2169 | else if (IS_GEN8(dev)) |
@@ -2030,9 +2181,6 @@ int intel_init_render_ring_buffer(struct drm_device *dev) | |||
2030 | 2181 | ||
2031 | /* Workaround batchbuffer to combat CS tlb bug. */ | 2182 | /* Workaround batchbuffer to combat CS tlb bug. */ |
2032 | if (HAS_BROKEN_CS_TLB(dev)) { | 2183 | if (HAS_BROKEN_CS_TLB(dev)) { |
2033 | struct drm_i915_gem_object *obj; | ||
2034 | int ret; | ||
2035 | |||
2036 | obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT); | 2184 | obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT); |
2037 | if (obj == NULL) { | 2185 | if (obj == NULL) { |
2038 | DRM_ERROR("Failed to allocate batch bo\n"); | 2186 | DRM_ERROR("Failed to allocate batch bo\n"); |
@@ -2163,31 +2311,32 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev) | |||
2163 | ring->irq_put = gen8_ring_put_irq; | 2311 | ring->irq_put = gen8_ring_put_irq; |
2164 | ring->dispatch_execbuffer = | 2312 | ring->dispatch_execbuffer = |
2165 | gen8_ring_dispatch_execbuffer; | 2313 | gen8_ring_dispatch_execbuffer; |
2314 | if (i915_semaphore_is_enabled(dev)) { | ||
2315 | ring->semaphore.sync_to = gen8_ring_sync; | ||
2316 | ring->semaphore.signal = gen8_xcs_signal; | ||
2317 | GEN8_RING_SEMAPHORE_INIT; | ||
2318 | } | ||
2166 | } else { | 2319 | } else { |
2167 | ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; | 2320 | ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; |
2168 | ring->irq_get = gen6_ring_get_irq; | 2321 | ring->irq_get = gen6_ring_get_irq; |
2169 | ring->irq_put = gen6_ring_put_irq; | 2322 | ring->irq_put = gen6_ring_put_irq; |
2170 | ring->dispatch_execbuffer = | 2323 | ring->dispatch_execbuffer = |
2171 | gen6_ring_dispatch_execbuffer; | 2324 | gen6_ring_dispatch_execbuffer; |
2325 | if (i915_semaphore_is_enabled(dev)) { | ||
2326 | ring->semaphore.sync_to = gen6_ring_sync; | ||
2327 | ring->semaphore.signal = gen6_signal; | ||
2328 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; | ||
2329 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2330 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; | ||
2331 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; | ||
2332 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | ||
2333 | ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; | ||
2334 | ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; | ||
2335 | ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; | ||
2336 | ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; | ||
2337 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2338 | } | ||
2172 | } | 2339 | } |
2173 | ring->semaphore.sync_to = gen6_ring_sync; | ||
2174 | ring->semaphore.signal = gen6_signal; | ||
2175 | /* | ||
2176 | * The current semaphore is only applied on pre-gen8 platform. | ||
2177 | * And there is no VCS2 ring on the pre-gen8 platform. So the | ||
2178 | * semaphore between VCS and VCS2 is initialized as INVALID. | ||
2179 | * Gen8 will initialize the sema between VCS2 and VCS later. | ||
2180 | */ | ||
2181 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; | ||
2182 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2183 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; | ||
2184 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; | ||
2185 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | ||
2186 | ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; | ||
2187 | ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; | ||
2188 | ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; | ||
2189 | ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; | ||
2190 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2191 | } else { | 2340 | } else { |
2192 | ring->mmio_base = BSD_RING_BASE; | 2341 | ring->mmio_base = BSD_RING_BASE; |
2193 | ring->flush = bsd_ring_flush; | 2342 | ring->flush = bsd_ring_flush; |
@@ -2224,7 +2373,7 @@ int intel_init_bsd2_ring_buffer(struct drm_device *dev) | |||
2224 | return -EINVAL; | 2373 | return -EINVAL; |
2225 | } | 2374 | } |
2226 | 2375 | ||
2227 | ring->name = "bds2_ring"; | 2376 | ring->name = "bsd2 ring"; |
2228 | ring->id = VCS2; | 2377 | ring->id = VCS2; |
2229 | 2378 | ||
2230 | ring->write_tail = ring_write_tail; | 2379 | ring->write_tail = ring_write_tail; |
@@ -2239,25 +2388,11 @@ int intel_init_bsd2_ring_buffer(struct drm_device *dev) | |||
2239 | ring->irq_put = gen8_ring_put_irq; | 2388 | ring->irq_put = gen8_ring_put_irq; |
2240 | ring->dispatch_execbuffer = | 2389 | ring->dispatch_execbuffer = |
2241 | gen8_ring_dispatch_execbuffer; | 2390 | gen8_ring_dispatch_execbuffer; |
2242 | ring->semaphore.sync_to = gen6_ring_sync; | 2391 | if (i915_semaphore_is_enabled(dev)) { |
2243 | ring->semaphore.signal = gen6_signal; | 2392 | ring->semaphore.sync_to = gen8_ring_sync; |
2244 | /* | 2393 | ring->semaphore.signal = gen8_xcs_signal; |
2245 | * The current semaphore is only applied on the pre-gen8. And there | 2394 | GEN8_RING_SEMAPHORE_INIT; |
2246 | * is no bsd2 ring on the pre-gen8. So now the semaphore_register | 2395 | } |
2247 | * between VCS2 and other ring is initialized as invalid. | ||
2248 | * Gen8 will initialize the sema between VCS2 and other ring later. | ||
2249 | */ | ||
2250 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2251 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2252 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2253 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2254 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | ||
2255 | ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; | ||
2256 | ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; | ||
2257 | ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; | ||
2258 | ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; | ||
2259 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2260 | |||
2261 | ring->init = init_ring_common; | 2396 | ring->init = init_ring_common; |
2262 | 2397 | ||
2263 | return intel_init_ring_buffer(dev, ring); | 2398 | return intel_init_ring_buffer(dev, ring); |
@@ -2283,30 +2418,38 @@ int intel_init_blt_ring_buffer(struct drm_device *dev) | |||
2283 | ring->irq_get = gen8_ring_get_irq; | 2418 | ring->irq_get = gen8_ring_get_irq; |
2284 | ring->irq_put = gen8_ring_put_irq; | 2419 | ring->irq_put = gen8_ring_put_irq; |
2285 | ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; | 2420 | ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
2421 | if (i915_semaphore_is_enabled(dev)) { | ||
2422 | ring->semaphore.sync_to = gen8_ring_sync; | ||
2423 | ring->semaphore.signal = gen8_xcs_signal; | ||
2424 | GEN8_RING_SEMAPHORE_INIT; | ||
2425 | } | ||
2286 | } else { | 2426 | } else { |
2287 | ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; | 2427 | ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; |
2288 | ring->irq_get = gen6_ring_get_irq; | 2428 | ring->irq_get = gen6_ring_get_irq; |
2289 | ring->irq_put = gen6_ring_put_irq; | 2429 | ring->irq_put = gen6_ring_put_irq; |
2290 | ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; | 2430 | ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
2431 | if (i915_semaphore_is_enabled(dev)) { | ||
2432 | ring->semaphore.signal = gen6_signal; | ||
2433 | ring->semaphore.sync_to = gen6_ring_sync; | ||
2434 | /* | ||
2435 | * The current semaphore is only applied on pre-gen8 | ||
2436 | * platform. And there is no VCS2 ring on the pre-gen8 | ||
2437 | * platform. So the semaphore between BCS and VCS2 is | ||
2438 | * initialized as INVALID. Gen8 will initialize the | ||
2439 | * sema between BCS and VCS2 later. | ||
2440 | */ | ||
2441 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; | ||
2442 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; | ||
2443 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2444 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; | ||
2445 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | ||
2446 | ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; | ||
2447 | ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; | ||
2448 | ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; | ||
2449 | ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; | ||
2450 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2451 | } | ||
2291 | } | 2452 | } |
2292 | ring->semaphore.sync_to = gen6_ring_sync; | ||
2293 | ring->semaphore.signal = gen6_signal; | ||
2294 | /* | ||
2295 | * The current semaphore is only applied on pre-gen8 platform. And | ||
2296 | * there is no VCS2 ring on the pre-gen8 platform. So the semaphore | ||
2297 | * between BCS and VCS2 is initialized as INVALID. | ||
2298 | * Gen8 will initialize the sema between BCS and VCS2 later. | ||
2299 | */ | ||
2300 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; | ||
2301 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; | ||
2302 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2303 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; | ||
2304 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | ||
2305 | ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; | ||
2306 | ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; | ||
2307 | ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; | ||
2308 | ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; | ||
2309 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2310 | ring->init = init_ring_common; | 2453 | ring->init = init_ring_common; |
2311 | 2454 | ||
2312 | return intel_init_ring_buffer(dev, ring); | 2455 | return intel_init_ring_buffer(dev, ring); |
@@ -2333,24 +2476,31 @@ int intel_init_vebox_ring_buffer(struct drm_device *dev) | |||
2333 | ring->irq_get = gen8_ring_get_irq; | 2476 | ring->irq_get = gen8_ring_get_irq; |
2334 | ring->irq_put = gen8_ring_put_irq; | 2477 | ring->irq_put = gen8_ring_put_irq; |
2335 | ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; | 2478 | ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
2479 | if (i915_semaphore_is_enabled(dev)) { | ||
2480 | ring->semaphore.sync_to = gen8_ring_sync; | ||
2481 | ring->semaphore.signal = gen8_xcs_signal; | ||
2482 | GEN8_RING_SEMAPHORE_INIT; | ||
2483 | } | ||
2336 | } else { | 2484 | } else { |
2337 | ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; | 2485 | ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; |
2338 | ring->irq_get = hsw_vebox_get_irq; | 2486 | ring->irq_get = hsw_vebox_get_irq; |
2339 | ring->irq_put = hsw_vebox_put_irq; | 2487 | ring->irq_put = hsw_vebox_put_irq; |
2340 | ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; | 2488 | ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
2489 | if (i915_semaphore_is_enabled(dev)) { | ||
2490 | ring->semaphore.sync_to = gen6_ring_sync; | ||
2491 | ring->semaphore.signal = gen6_signal; | ||
2492 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; | ||
2493 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; | ||
2494 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; | ||
2495 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2496 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | ||
2497 | ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; | ||
2498 | ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; | ||
2499 | ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; | ||
2500 | ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; | ||
2501 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2502 | } | ||
2341 | } | 2503 | } |
2342 | ring->semaphore.sync_to = gen6_ring_sync; | ||
2343 | ring->semaphore.signal = gen6_signal; | ||
2344 | ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; | ||
2345 | ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; | ||
2346 | ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; | ||
2347 | ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; | ||
2348 | ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; | ||
2349 | ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; | ||
2350 | ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; | ||
2351 | ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; | ||
2352 | ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; | ||
2353 | ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; | ||
2354 | ring->init = init_ring_common; | 2504 | ring->init = init_ring_common; |
2355 | 2505 | ||
2356 | return intel_init_ring_buffer(dev, ring); | 2506 | return intel_init_ring_buffer(dev, ring); |