aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/intel_ringbuffer.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/intel_ringbuffer.c')
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c1323
1 files changed, 577 insertions, 746 deletions
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 68c5af079ef8..cca7792f26d5 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -34,6 +34,11 @@
34#include "i915_trace.h" 34#include "i915_trace.h"
35#include "intel_drv.h" 35#include "intel_drv.h"
36 36
37/* Rough estimate of the typical request size, performing a flush,
38 * set-context and then emitting the batch.
39 */
40#define LEGACY_REQUEST_SIZE 200
41
37int __intel_ring_space(int head, int tail, int size) 42int __intel_ring_space(int head, int tail, int size)
38{ 43{
39 int space = head - tail; 44 int space = head - tail;
@@ -53,18 +58,10 @@ void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
53 ringbuf->tail, ringbuf->size); 58 ringbuf->tail, ringbuf->size);
54} 59}
55 60
56bool intel_engine_stopped(struct intel_engine_cs *engine)
57{
58 struct drm_i915_private *dev_priv = engine->dev->dev_private;
59 return dev_priv->gpu_error.stop_rings & intel_engine_flag(engine);
60}
61
62static void __intel_ring_advance(struct intel_engine_cs *engine) 61static void __intel_ring_advance(struct intel_engine_cs *engine)
63{ 62{
64 struct intel_ringbuffer *ringbuf = engine->buffer; 63 struct intel_ringbuffer *ringbuf = engine->buffer;
65 ringbuf->tail &= ringbuf->size - 1; 64 ringbuf->tail &= ringbuf->size - 1;
66 if (intel_engine_stopped(engine))
67 return;
68 engine->write_tail(engine, ringbuf->tail); 65 engine->write_tail(engine, ringbuf->tail);
69} 66}
70 67
@@ -101,7 +98,6 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req,
101 u32 flush_domains) 98 u32 flush_domains)
102{ 99{
103 struct intel_engine_cs *engine = req->engine; 100 struct intel_engine_cs *engine = req->engine;
104 struct drm_device *dev = engine->dev;
105 u32 cmd; 101 u32 cmd;
106 int ret; 102 int ret;
107 103
@@ -140,7 +136,7 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req,
140 cmd |= MI_EXE_FLUSH; 136 cmd |= MI_EXE_FLUSH;
141 137
142 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND && 138 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
143 (IS_G4X(dev) || IS_GEN5(dev))) 139 (IS_G4X(req->i915) || IS_GEN5(req->i915)))
144 cmd |= MI_INVALIDATE_ISP; 140 cmd |= MI_INVALIDATE_ISP;
145 141
146 ret = intel_ring_begin(req, 2); 142 ret = intel_ring_begin(req, 2);
@@ -426,19 +422,19 @@ gen8_render_ring_flush(struct drm_i915_gem_request *req,
426static void ring_write_tail(struct intel_engine_cs *engine, 422static void ring_write_tail(struct intel_engine_cs *engine,
427 u32 value) 423 u32 value)
428{ 424{
429 struct drm_i915_private *dev_priv = engine->dev->dev_private; 425 struct drm_i915_private *dev_priv = engine->i915;
430 I915_WRITE_TAIL(engine, value); 426 I915_WRITE_TAIL(engine, value);
431} 427}
432 428
433u64 intel_ring_get_active_head(struct intel_engine_cs *engine) 429u64 intel_ring_get_active_head(struct intel_engine_cs *engine)
434{ 430{
435 struct drm_i915_private *dev_priv = engine->dev->dev_private; 431 struct drm_i915_private *dev_priv = engine->i915;
436 u64 acthd; 432 u64 acthd;
437 433
438 if (INTEL_INFO(engine->dev)->gen >= 8) 434 if (INTEL_GEN(dev_priv) >= 8)
439 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base), 435 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
440 RING_ACTHD_UDW(engine->mmio_base)); 436 RING_ACTHD_UDW(engine->mmio_base));
441 else if (INTEL_INFO(engine->dev)->gen >= 4) 437 else if (INTEL_GEN(dev_priv) >= 4)
442 acthd = I915_READ(RING_ACTHD(engine->mmio_base)); 438 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
443 else 439 else
444 acthd = I915_READ(ACTHD); 440 acthd = I915_READ(ACTHD);
@@ -448,25 +444,24 @@ u64 intel_ring_get_active_head(struct intel_engine_cs *engine)
448 444
449static void ring_setup_phys_status_page(struct intel_engine_cs *engine) 445static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
450{ 446{
451 struct drm_i915_private *dev_priv = engine->dev->dev_private; 447 struct drm_i915_private *dev_priv = engine->i915;
452 u32 addr; 448 u32 addr;
453 449
454 addr = dev_priv->status_page_dmah->busaddr; 450 addr = dev_priv->status_page_dmah->busaddr;
455 if (INTEL_INFO(engine->dev)->gen >= 4) 451 if (INTEL_GEN(dev_priv) >= 4)
456 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; 452 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
457 I915_WRITE(HWS_PGA, addr); 453 I915_WRITE(HWS_PGA, addr);
458} 454}
459 455
460static void intel_ring_setup_status_page(struct intel_engine_cs *engine) 456static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
461{ 457{
462 struct drm_device *dev = engine->dev; 458 struct drm_i915_private *dev_priv = engine->i915;
463 struct drm_i915_private *dev_priv = engine->dev->dev_private;
464 i915_reg_t mmio; 459 i915_reg_t mmio;
465 460
466 /* The ring status page addresses are no longer next to the rest of 461 /* The ring status page addresses are no longer next to the rest of
467 * the ring registers as of gen7. 462 * the ring registers as of gen7.
468 */ 463 */
469 if (IS_GEN7(dev)) { 464 if (IS_GEN7(dev_priv)) {
470 switch (engine->id) { 465 switch (engine->id) {
471 case RCS: 466 case RCS:
472 mmio = RENDER_HWS_PGA_GEN7; 467 mmio = RENDER_HWS_PGA_GEN7;
@@ -486,7 +481,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
486 mmio = VEBOX_HWS_PGA_GEN7; 481 mmio = VEBOX_HWS_PGA_GEN7;
487 break; 482 break;
488 } 483 }
489 } else if (IS_GEN6(engine->dev)) { 484 } else if (IS_GEN6(dev_priv)) {
490 mmio = RING_HWS_PGA_GEN6(engine->mmio_base); 485 mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
491 } else { 486 } else {
492 /* XXX: gen8 returns to sanity */ 487 /* XXX: gen8 returns to sanity */
@@ -503,7 +498,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
503 * arises: do we still need this and if so how should we go about 498 * arises: do we still need this and if so how should we go about
504 * invalidating the TLB? 499 * invalidating the TLB?
505 */ 500 */
506 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) { 501 if (IS_GEN(dev_priv, 6, 7)) {
507 i915_reg_t reg = RING_INSTPM(engine->mmio_base); 502 i915_reg_t reg = RING_INSTPM(engine->mmio_base);
508 503
509 /* ring should be idle before issuing a sync flush*/ 504 /* ring should be idle before issuing a sync flush*/
@@ -512,8 +507,9 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
512 I915_WRITE(reg, 507 I915_WRITE(reg,
513 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 508 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
514 INSTPM_SYNC_FLUSH)); 509 INSTPM_SYNC_FLUSH));
515 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, 510 if (intel_wait_for_register(dev_priv,
516 1000)) 511 reg, INSTPM_SYNC_FLUSH, 0,
512 1000))
517 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", 513 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
518 engine->name); 514 engine->name);
519 } 515 }
@@ -521,11 +517,15 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
521 517
522static bool stop_ring(struct intel_engine_cs *engine) 518static bool stop_ring(struct intel_engine_cs *engine)
523{ 519{
524 struct drm_i915_private *dev_priv = to_i915(engine->dev); 520 struct drm_i915_private *dev_priv = engine->i915;
525 521
526 if (!IS_GEN2(engine->dev)) { 522 if (!IS_GEN2(dev_priv)) {
527 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); 523 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
528 if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) { 524 if (intel_wait_for_register(dev_priv,
525 RING_MI_MODE(engine->mmio_base),
526 MODE_IDLE,
527 MODE_IDLE,
528 1000)) {
529 DRM_ERROR("%s : timed out trying to stop ring\n", 529 DRM_ERROR("%s : timed out trying to stop ring\n",
530 engine->name); 530 engine->name);
531 /* Sometimes we observe that the idle flag is not 531 /* Sometimes we observe that the idle flag is not
@@ -541,7 +541,7 @@ static bool stop_ring(struct intel_engine_cs *engine)
541 I915_WRITE_HEAD(engine, 0); 541 I915_WRITE_HEAD(engine, 0);
542 engine->write_tail(engine, 0); 542 engine->write_tail(engine, 0);
543 543
544 if (!IS_GEN2(engine->dev)) { 544 if (!IS_GEN2(dev_priv)) {
545 (void)I915_READ_CTL(engine); 545 (void)I915_READ_CTL(engine);
546 I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING)); 546 I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
547 } 547 }
@@ -556,8 +556,7 @@ void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
556 556
557static int init_ring_common(struct intel_engine_cs *engine) 557static int init_ring_common(struct intel_engine_cs *engine)
558{ 558{
559 struct drm_device *dev = engine->dev; 559 struct drm_i915_private *dev_priv = engine->i915;
560 struct drm_i915_private *dev_priv = dev->dev_private;
561 struct intel_ringbuffer *ringbuf = engine->buffer; 560 struct intel_ringbuffer *ringbuf = engine->buffer;
562 struct drm_i915_gem_object *obj = ringbuf->obj; 561 struct drm_i915_gem_object *obj = ringbuf->obj;
563 int ret = 0; 562 int ret = 0;
@@ -587,7 +586,7 @@ static int init_ring_common(struct intel_engine_cs *engine)
587 } 586 }
588 } 587 }
589 588
590 if (I915_NEED_GFX_HWS(dev)) 589 if (I915_NEED_GFX_HWS(dev_priv))
591 intel_ring_setup_status_page(engine); 590 intel_ring_setup_status_page(engine);
592 else 591 else
593 ring_setup_phys_status_page(engine); 592 ring_setup_phys_status_page(engine);
@@ -641,59 +640,42 @@ out:
641 return ret; 640 return ret;
642} 641}
643 642
644void 643void intel_fini_pipe_control(struct intel_engine_cs *engine)
645intel_fini_pipe_control(struct intel_engine_cs *engine)
646{ 644{
647 struct drm_device *dev = engine->dev;
648
649 if (engine->scratch.obj == NULL) 645 if (engine->scratch.obj == NULL)
650 return; 646 return;
651 647
652 if (INTEL_INFO(dev)->gen >= 5) { 648 i915_gem_object_ggtt_unpin(engine->scratch.obj);
653 kunmap(sg_page(engine->scratch.obj->pages->sgl));
654 i915_gem_object_ggtt_unpin(engine->scratch.obj);
655 }
656
657 drm_gem_object_unreference(&engine->scratch.obj->base); 649 drm_gem_object_unreference(&engine->scratch.obj->base);
658 engine->scratch.obj = NULL; 650 engine->scratch.obj = NULL;
659} 651}
660 652
661int 653int intel_init_pipe_control(struct intel_engine_cs *engine, int size)
662intel_init_pipe_control(struct intel_engine_cs *engine)
663{ 654{
655 struct drm_i915_gem_object *obj;
664 int ret; 656 int ret;
665 657
666 WARN_ON(engine->scratch.obj); 658 WARN_ON(engine->scratch.obj);
667 659
668 engine->scratch.obj = i915_gem_alloc_object(engine->dev, 4096); 660 obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
669 if (engine->scratch.obj == NULL) { 661 if (!obj)
670 DRM_ERROR("Failed to allocate seqno page\n"); 662 obj = i915_gem_object_create(&engine->i915->drm, size);
671 ret = -ENOMEM; 663 if (IS_ERR(obj)) {
664 DRM_ERROR("Failed to allocate scratch page\n");
665 ret = PTR_ERR(obj);
672 goto err; 666 goto err;
673 } 667 }
674 668
675 ret = i915_gem_object_set_cache_level(engine->scratch.obj, 669 ret = i915_gem_obj_ggtt_pin(obj, 4096, PIN_HIGH);
676 I915_CACHE_LLC);
677 if (ret)
678 goto err_unref;
679
680 ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0);
681 if (ret) 670 if (ret)
682 goto err_unref; 671 goto err_unref;
683 672
684 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj); 673 engine->scratch.obj = obj;
685 engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl)); 674 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
686 if (engine->scratch.cpu_page == NULL) {
687 ret = -ENOMEM;
688 goto err_unpin;
689 }
690
691 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n", 675 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
692 engine->name, engine->scratch.gtt_offset); 676 engine->name, engine->scratch.gtt_offset);
693 return 0; 677 return 0;
694 678
695err_unpin:
696 i915_gem_object_ggtt_unpin(engine->scratch.obj);
697err_unref: 679err_unref:
698 drm_gem_object_unreference(&engine->scratch.obj->base); 680 drm_gem_object_unreference(&engine->scratch.obj->base);
699err: 681err:
@@ -702,11 +684,9 @@ err:
702 684
703static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req) 685static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
704{ 686{
705 int ret, i;
706 struct intel_engine_cs *engine = req->engine; 687 struct intel_engine_cs *engine = req->engine;
707 struct drm_device *dev = engine->dev; 688 struct i915_workarounds *w = &req->i915->workarounds;
708 struct drm_i915_private *dev_priv = dev->dev_private; 689 int ret, i;
709 struct i915_workarounds *w = &dev_priv->workarounds;
710 690
711 if (w->count == 0) 691 if (w->count == 0)
712 return 0; 692 return 0;
@@ -795,7 +775,7 @@ static int wa_add(struct drm_i915_private *dev_priv,
795static int wa_ring_whitelist_reg(struct intel_engine_cs *engine, 775static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
796 i915_reg_t reg) 776 i915_reg_t reg)
797{ 777{
798 struct drm_i915_private *dev_priv = engine->dev->dev_private; 778 struct drm_i915_private *dev_priv = engine->i915;
799 struct i915_workarounds *wa = &dev_priv->workarounds; 779 struct i915_workarounds *wa = &dev_priv->workarounds;
800 const uint32_t index = wa->hw_whitelist_count[engine->id]; 780 const uint32_t index = wa->hw_whitelist_count[engine->id];
801 781
@@ -811,8 +791,7 @@ static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
811 791
812static int gen8_init_workarounds(struct intel_engine_cs *engine) 792static int gen8_init_workarounds(struct intel_engine_cs *engine)
813{ 793{
814 struct drm_device *dev = engine->dev; 794 struct drm_i915_private *dev_priv = engine->i915;
815 struct drm_i915_private *dev_priv = dev->dev_private;
816 795
817 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING); 796 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
818 797
@@ -863,9 +842,8 @@ static int gen8_init_workarounds(struct intel_engine_cs *engine)
863 842
864static int bdw_init_workarounds(struct intel_engine_cs *engine) 843static int bdw_init_workarounds(struct intel_engine_cs *engine)
865{ 844{
845 struct drm_i915_private *dev_priv = engine->i915;
866 int ret; 846 int ret;
867 struct drm_device *dev = engine->dev;
868 struct drm_i915_private *dev_priv = dev->dev_private;
869 847
870 ret = gen8_init_workarounds(engine); 848 ret = gen8_init_workarounds(engine);
871 if (ret) 849 if (ret)
@@ -885,16 +863,15 @@ static int bdw_init_workarounds(struct intel_engine_cs *engine)
885 /* WaForceContextSaveRestoreNonCoherent:bdw */ 863 /* WaForceContextSaveRestoreNonCoherent:bdw */
886 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | 864 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
887 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ 865 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
888 (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); 866 (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
889 867
890 return 0; 868 return 0;
891} 869}
892 870
893static int chv_init_workarounds(struct intel_engine_cs *engine) 871static int chv_init_workarounds(struct intel_engine_cs *engine)
894{ 872{
873 struct drm_i915_private *dev_priv = engine->i915;
895 int ret; 874 int ret;
896 struct drm_device *dev = engine->dev;
897 struct drm_i915_private *dev_priv = dev->dev_private;
898 875
899 ret = gen8_init_workarounds(engine); 876 ret = gen8_init_workarounds(engine);
900 if (ret) 877 if (ret)
@@ -911,8 +888,7 @@ static int chv_init_workarounds(struct intel_engine_cs *engine)
911 888
912static int gen9_init_workarounds(struct intel_engine_cs *engine) 889static int gen9_init_workarounds(struct intel_engine_cs *engine)
913{ 890{
914 struct drm_device *dev = engine->dev; 891 struct drm_i915_private *dev_priv = engine->i915;
915 struct drm_i915_private *dev_priv = dev->dev_private;
916 int ret; 892 int ret;
917 893
918 /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl */ 894 /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl */
@@ -937,14 +913,14 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
937 GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC); 913 GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
938 914
939 /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */ 915 /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
940 if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || 916 if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
941 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) 917 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
942 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5, 918 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
943 GEN9_DG_MIRROR_FIX_ENABLE); 919 GEN9_DG_MIRROR_FIX_ENABLE);
944 920
945 /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ 921 /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
946 if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || 922 if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
947 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { 923 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
948 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1, 924 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
949 GEN9_RHWO_OPTIMIZATION_DISABLE); 925 GEN9_RHWO_OPTIMIZATION_DISABLE);
950 /* 926 /*
@@ -970,8 +946,8 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
970 GEN9_CCS_TLB_PREFETCH_ENABLE); 946 GEN9_CCS_TLB_PREFETCH_ENABLE);
971 947
972 /* WaDisableMaskBasedCammingInRCC:skl,bxt */ 948 /* WaDisableMaskBasedCammingInRCC:skl,bxt */
973 if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) || 949 if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_C0) ||
974 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) 950 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
975 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0, 951 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
976 PIXEL_MASK_CAMMING_DISABLE); 952 PIXEL_MASK_CAMMING_DISABLE);
977 953
@@ -1035,8 +1011,7 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
1035 1011
1036static int skl_tune_iz_hashing(struct intel_engine_cs *engine) 1012static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1037{ 1013{
1038 struct drm_device *dev = engine->dev; 1014 struct drm_i915_private *dev_priv = engine->i915;
1039 struct drm_i915_private *dev_priv = dev->dev_private;
1040 u8 vals[3] = { 0, 0, 0 }; 1015 u8 vals[3] = { 0, 0, 0 };
1041 unsigned int i; 1016 unsigned int i;
1042 1017
@@ -1077,9 +1052,8 @@ static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1077 1052
1078static int skl_init_workarounds(struct intel_engine_cs *engine) 1053static int skl_init_workarounds(struct intel_engine_cs *engine)
1079{ 1054{
1055 struct drm_i915_private *dev_priv = engine->i915;
1080 int ret; 1056 int ret;
1081 struct drm_device *dev = engine->dev;
1082 struct drm_i915_private *dev_priv = dev->dev_private;
1083 1057
1084 ret = gen9_init_workarounds(engine); 1058 ret = gen9_init_workarounds(engine);
1085 if (ret) 1059 if (ret)
@@ -1090,12 +1064,12 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
1090 * until D0 which is the default case so this is equivalent to 1064 * until D0 which is the default case so this is equivalent to
1091 * !WaDisablePerCtxtPreemptionGranularityControl:skl 1065 * !WaDisablePerCtxtPreemptionGranularityControl:skl
1092 */ 1066 */
1093 if (IS_SKL_REVID(dev, SKL_REVID_E0, REVID_FOREVER)) { 1067 if (IS_SKL_REVID(dev_priv, SKL_REVID_E0, REVID_FOREVER)) {
1094 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1, 1068 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1095 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL)); 1069 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1096 } 1070 }
1097 1071
1098 if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) { 1072 if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0)) {
1099 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */ 1073 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
1100 I915_WRITE(FF_SLICE_CS_CHICKEN2, 1074 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1101 _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE)); 1075 _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
@@ -1104,30 +1078,30 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
1104 /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes 1078 /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
1105 * involving this register should also be added to WA batch as required. 1079 * involving this register should also be added to WA batch as required.
1106 */ 1080 */
1107 if (IS_SKL_REVID(dev, 0, SKL_REVID_E0)) 1081 if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0))
1108 /* WaDisableLSQCROPERFforOCL:skl */ 1082 /* WaDisableLSQCROPERFforOCL:skl */
1109 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | 1083 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1110 GEN8_LQSC_RO_PERF_DIS); 1084 GEN8_LQSC_RO_PERF_DIS);
1111 1085
1112 /* WaEnableGapsTsvCreditFix:skl */ 1086 /* WaEnableGapsTsvCreditFix:skl */
1113 if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) { 1087 if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, REVID_FOREVER)) {
1114 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) | 1088 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1115 GEN9_GAPS_TSV_CREDIT_DISABLE)); 1089 GEN9_GAPS_TSV_CREDIT_DISABLE));
1116 } 1090 }
1117 1091
1118 /* WaDisablePowerCompilerClockGating:skl */ 1092 /* WaDisablePowerCompilerClockGating:skl */
1119 if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0)) 1093 if (IS_SKL_REVID(dev_priv, SKL_REVID_B0, SKL_REVID_B0))
1120 WA_SET_BIT_MASKED(HIZ_CHICKEN, 1094 WA_SET_BIT_MASKED(HIZ_CHICKEN,
1121 BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE); 1095 BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
1122 1096
1123 /* WaBarrierPerformanceFixDisable:skl */ 1097 /* WaBarrierPerformanceFixDisable:skl */
1124 if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0)) 1098 if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_D0))
1125 WA_SET_BIT_MASKED(HDC_CHICKEN0, 1099 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1126 HDC_FENCE_DEST_SLM_DISABLE | 1100 HDC_FENCE_DEST_SLM_DISABLE |
1127 HDC_BARRIER_PERFORMANCE_DISABLE); 1101 HDC_BARRIER_PERFORMANCE_DISABLE);
1128 1102
1129 /* WaDisableSbeCacheDispatchPortSharing:skl */ 1103 /* WaDisableSbeCacheDispatchPortSharing:skl */
1130 if (IS_SKL_REVID(dev, 0, SKL_REVID_F0)) 1104 if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0))
1131 WA_SET_BIT_MASKED( 1105 WA_SET_BIT_MASKED(
1132 GEN7_HALF_SLICE_CHICKEN1, 1106 GEN7_HALF_SLICE_CHICKEN1,
1133 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); 1107 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
@@ -1135,6 +1109,11 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
1135 /* WaDisableGafsUnitClkGating:skl */ 1109 /* WaDisableGafsUnitClkGating:skl */
1136 WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 1110 WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1137 1111
1112 /* WaInPlaceDecompressionHang:skl */
1113 if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
1114 WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
1115 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1116
1138 /* WaDisableLSQCROPERFforOCL:skl */ 1117 /* WaDisableLSQCROPERFforOCL:skl */
1139 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4); 1118 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1140 if (ret) 1119 if (ret)
@@ -1145,9 +1124,8 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
1145 1124
1146static int bxt_init_workarounds(struct intel_engine_cs *engine) 1125static int bxt_init_workarounds(struct intel_engine_cs *engine)
1147{ 1126{
1127 struct drm_i915_private *dev_priv = engine->i915;
1148 int ret; 1128 int ret;
1149 struct drm_device *dev = engine->dev;
1150 struct drm_i915_private *dev_priv = dev->dev_private;
1151 1129
1152 ret = gen9_init_workarounds(engine); 1130 ret = gen9_init_workarounds(engine);
1153 if (ret) 1131 if (ret)
@@ -1155,11 +1133,11 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
1155 1133
1156 /* WaStoreMultiplePTEenable:bxt */ 1134 /* WaStoreMultiplePTEenable:bxt */
1157 /* This is a requirement according to Hardware specification */ 1135 /* This is a requirement according to Hardware specification */
1158 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) 1136 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1159 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF); 1137 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1160 1138
1161 /* WaSetClckGatingDisableMedia:bxt */ 1139 /* WaSetClckGatingDisableMedia:bxt */
1162 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { 1140 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1163 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & 1141 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1164 ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE)); 1142 ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1165 } 1143 }
@@ -1168,8 +1146,14 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
1168 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, 1146 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1169 STALL_DOP_GATING_DISABLE); 1147 STALL_DOP_GATING_DISABLE);
1170 1148
1149 /* WaDisablePooledEuLoadBalancingFix:bxt */
1150 if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1151 WA_SET_BIT_MASKED(FF_SLICE_CS_CHICKEN2,
1152 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1153 }
1154
1171 /* WaDisableSbeCacheDispatchPortSharing:bxt */ 1155 /* WaDisableSbeCacheDispatchPortSharing:bxt */
1172 if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) { 1156 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
1173 WA_SET_BIT_MASKED( 1157 WA_SET_BIT_MASKED(
1174 GEN7_HALF_SLICE_CHICKEN1, 1158 GEN7_HALF_SLICE_CHICKEN1,
1175 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); 1159 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
@@ -1179,7 +1163,7 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
1179 /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */ 1163 /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
1180 /* WaDisableObjectLevelPreemtionForInstanceId:bxt */ 1164 /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
1181 /* WaDisableLSQCROPERFforOCL:bxt */ 1165 /* WaDisableLSQCROPERFforOCL:bxt */
1182 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { 1166 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1183 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1); 1167 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1184 if (ret) 1168 if (ret)
1185 return ret; 1169 return ret;
@@ -1189,17 +1173,27 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
1189 return ret; 1173 return ret;
1190 } 1174 }
1191 1175
1176 /* WaProgramL3SqcReg1DefaultForPerf:bxt */
1177 if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
1178 I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) |
1179 L3_HIGH_PRIO_CREDITS(2));
1180
1192 /* WaInsertDummyPushConstPs:bxt */ 1181 /* WaInsertDummyPushConstPs:bxt */
1193 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) 1182 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1194 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, 1183 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1195 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 1184 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1196 1185
1186 /* WaInPlaceDecompressionHang:bxt */
1187 if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1188 WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
1189 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1190
1197 return 0; 1191 return 0;
1198} 1192}
1199 1193
1200static int kbl_init_workarounds(struct intel_engine_cs *engine) 1194static int kbl_init_workarounds(struct intel_engine_cs *engine)
1201{ 1195{
1202 struct drm_i915_private *dev_priv = engine->dev->dev_private; 1196 struct drm_i915_private *dev_priv = engine->i915;
1203 int ret; 1197 int ret;
1204 1198
1205 ret = gen9_init_workarounds(engine); 1199 ret = gen9_init_workarounds(engine);
@@ -1241,6 +1235,10 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine)
1241 GEN7_HALF_SLICE_CHICKEN1, 1235 GEN7_HALF_SLICE_CHICKEN1,
1242 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); 1236 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1243 1237
1238 /* WaInPlaceDecompressionHang:kbl */
1239 WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
1240 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1241
1244 /* WaDisableLSQCROPERFforOCL:kbl */ 1242 /* WaDisableLSQCROPERFforOCL:kbl */
1245 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4); 1243 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1246 if (ret) 1244 if (ret)
@@ -1251,24 +1249,23 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine)
1251 1249
1252int init_workarounds_ring(struct intel_engine_cs *engine) 1250int init_workarounds_ring(struct intel_engine_cs *engine)
1253{ 1251{
1254 struct drm_device *dev = engine->dev; 1252 struct drm_i915_private *dev_priv = engine->i915;
1255 struct drm_i915_private *dev_priv = dev->dev_private;
1256 1253
1257 WARN_ON(engine->id != RCS); 1254 WARN_ON(engine->id != RCS);
1258 1255
1259 dev_priv->workarounds.count = 0; 1256 dev_priv->workarounds.count = 0;
1260 dev_priv->workarounds.hw_whitelist_count[RCS] = 0; 1257 dev_priv->workarounds.hw_whitelist_count[RCS] = 0;
1261 1258
1262 if (IS_BROADWELL(dev)) 1259 if (IS_BROADWELL(dev_priv))
1263 return bdw_init_workarounds(engine); 1260 return bdw_init_workarounds(engine);
1264 1261
1265 if (IS_CHERRYVIEW(dev)) 1262 if (IS_CHERRYVIEW(dev_priv))
1266 return chv_init_workarounds(engine); 1263 return chv_init_workarounds(engine);
1267 1264
1268 if (IS_SKYLAKE(dev)) 1265 if (IS_SKYLAKE(dev_priv))
1269 return skl_init_workarounds(engine); 1266 return skl_init_workarounds(engine);
1270 1267
1271 if (IS_BROXTON(dev)) 1268 if (IS_BROXTON(dev_priv))
1272 return bxt_init_workarounds(engine); 1269 return bxt_init_workarounds(engine);
1273 1270
1274 if (IS_KABYLAKE(dev_priv)) 1271 if (IS_KABYLAKE(dev_priv))
@@ -1279,14 +1276,13 @@ int init_workarounds_ring(struct intel_engine_cs *engine)
1279 1276
1280static int init_render_ring(struct intel_engine_cs *engine) 1277static int init_render_ring(struct intel_engine_cs *engine)
1281{ 1278{
1282 struct drm_device *dev = engine->dev; 1279 struct drm_i915_private *dev_priv = engine->i915;
1283 struct drm_i915_private *dev_priv = dev->dev_private;
1284 int ret = init_ring_common(engine); 1280 int ret = init_ring_common(engine);
1285 if (ret) 1281 if (ret)
1286 return ret; 1282 return ret;
1287 1283
1288 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ 1284 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1289 if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7) 1285 if (IS_GEN(dev_priv, 4, 6))
1290 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); 1286 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
1291 1287
1292 /* We need to disable the AsyncFlip performance optimisations in order 1288 /* We need to disable the AsyncFlip performance optimisations in order
@@ -1295,22 +1291,22 @@ static int init_render_ring(struct intel_engine_cs *engine)
1295 * 1291 *
1296 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv 1292 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1297 */ 1293 */
1298 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) 1294 if (IS_GEN(dev_priv, 6, 7))
1299 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); 1295 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1300 1296
1301 /* Required for the hardware to program scanline values for waiting */ 1297 /* Required for the hardware to program scanline values for waiting */
1302 /* WaEnableFlushTlbInvalidationMode:snb */ 1298 /* WaEnableFlushTlbInvalidationMode:snb */
1303 if (INTEL_INFO(dev)->gen == 6) 1299 if (IS_GEN6(dev_priv))
1304 I915_WRITE(GFX_MODE, 1300 I915_WRITE(GFX_MODE,
1305 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT)); 1301 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
1306 1302
1307 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ 1303 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1308 if (IS_GEN7(dev)) 1304 if (IS_GEN7(dev_priv))
1309 I915_WRITE(GFX_MODE_GEN7, 1305 I915_WRITE(GFX_MODE_GEN7,
1310 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) | 1306 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
1311 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); 1307 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
1312 1308
1313 if (IS_GEN6(dev)) { 1309 if (IS_GEN6(dev_priv)) {
1314 /* From the Sandybridge PRM, volume 1 part 3, page 24: 1310 /* From the Sandybridge PRM, volume 1 part 3, page 24:
1315 * "If this bit is set, STCunit will have LRA as replacement 1311 * "If this bit is set, STCunit will have LRA as replacement
1316 * policy. [...] This bit must be reset. LRA replacement 1312 * policy. [...] This bit must be reset. LRA replacement
@@ -1320,19 +1316,18 @@ static int init_render_ring(struct intel_engine_cs *engine)
1320 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); 1316 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
1321 } 1317 }
1322 1318
1323 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) 1319 if (IS_GEN(dev_priv, 6, 7))
1324 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 1320 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1325 1321
1326 if (HAS_L3_DPF(dev)) 1322 if (INTEL_INFO(dev_priv)->gen >= 6)
1327 I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev)); 1323 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1328 1324
1329 return init_workarounds_ring(engine); 1325 return init_workarounds_ring(engine);
1330} 1326}
1331 1327
1332static void render_ring_cleanup(struct intel_engine_cs *engine) 1328static void render_ring_cleanup(struct intel_engine_cs *engine)
1333{ 1329{
1334 struct drm_device *dev = engine->dev; 1330 struct drm_i915_private *dev_priv = engine->i915;
1335 struct drm_i915_private *dev_priv = dev->dev_private;
1336 1331
1337 if (dev_priv->semaphore_obj) { 1332 if (dev_priv->semaphore_obj) {
1338 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj); 1333 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
@@ -1348,13 +1343,12 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
1348{ 1343{
1349#define MBOX_UPDATE_DWORDS 8 1344#define MBOX_UPDATE_DWORDS 8
1350 struct intel_engine_cs *signaller = signaller_req->engine; 1345 struct intel_engine_cs *signaller = signaller_req->engine;
1351 struct drm_device *dev = signaller->dev; 1346 struct drm_i915_private *dev_priv = signaller_req->i915;
1352 struct drm_i915_private *dev_priv = dev->dev_private;
1353 struct intel_engine_cs *waiter; 1347 struct intel_engine_cs *waiter;
1354 enum intel_engine_id id; 1348 enum intel_engine_id id;
1355 int ret, num_rings; 1349 int ret, num_rings;
1356 1350
1357 num_rings = hweight32(INTEL_INFO(dev)->ring_mask); 1351 num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
1358 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; 1352 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1359#undef MBOX_UPDATE_DWORDS 1353#undef MBOX_UPDATE_DWORDS
1360 1354
@@ -1363,19 +1357,17 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
1363 return ret; 1357 return ret;
1364 1358
1365 for_each_engine_id(waiter, dev_priv, id) { 1359 for_each_engine_id(waiter, dev_priv, id) {
1366 u32 seqno;
1367 u64 gtt_offset = signaller->semaphore.signal_ggtt[id]; 1360 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1368 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) 1361 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1369 continue; 1362 continue;
1370 1363
1371 seqno = i915_gem_request_get_seqno(signaller_req);
1372 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6)); 1364 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
1373 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB | 1365 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
1374 PIPE_CONTROL_QW_WRITE | 1366 PIPE_CONTROL_QW_WRITE |
1375 PIPE_CONTROL_FLUSH_ENABLE); 1367 PIPE_CONTROL_CS_STALL);
1376 intel_ring_emit(signaller, lower_32_bits(gtt_offset)); 1368 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
1377 intel_ring_emit(signaller, upper_32_bits(gtt_offset)); 1369 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1378 intel_ring_emit(signaller, seqno); 1370 intel_ring_emit(signaller, signaller_req->seqno);
1379 intel_ring_emit(signaller, 0); 1371 intel_ring_emit(signaller, 0);
1380 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | 1372 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1381 MI_SEMAPHORE_TARGET(waiter->hw_id)); 1373 MI_SEMAPHORE_TARGET(waiter->hw_id));
@@ -1390,13 +1382,12 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
1390{ 1382{
1391#define MBOX_UPDATE_DWORDS 6 1383#define MBOX_UPDATE_DWORDS 6
1392 struct intel_engine_cs *signaller = signaller_req->engine; 1384 struct intel_engine_cs *signaller = signaller_req->engine;
1393 struct drm_device *dev = signaller->dev; 1385 struct drm_i915_private *dev_priv = signaller_req->i915;
1394 struct drm_i915_private *dev_priv = dev->dev_private;
1395 struct intel_engine_cs *waiter; 1386 struct intel_engine_cs *waiter;
1396 enum intel_engine_id id; 1387 enum intel_engine_id id;
1397 int ret, num_rings; 1388 int ret, num_rings;
1398 1389
1399 num_rings = hweight32(INTEL_INFO(dev)->ring_mask); 1390 num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
1400 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; 1391 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1401#undef MBOX_UPDATE_DWORDS 1392#undef MBOX_UPDATE_DWORDS
1402 1393
@@ -1405,18 +1396,16 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
1405 return ret; 1396 return ret;
1406 1397
1407 for_each_engine_id(waiter, dev_priv, id) { 1398 for_each_engine_id(waiter, dev_priv, id) {
1408 u32 seqno;
1409 u64 gtt_offset = signaller->semaphore.signal_ggtt[id]; 1399 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1410 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) 1400 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1411 continue; 1401 continue;
1412 1402
1413 seqno = i915_gem_request_get_seqno(signaller_req);
1414 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) | 1403 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
1415 MI_FLUSH_DW_OP_STOREDW); 1404 MI_FLUSH_DW_OP_STOREDW);
1416 intel_ring_emit(signaller, lower_32_bits(gtt_offset) | 1405 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
1417 MI_FLUSH_DW_USE_GTT); 1406 MI_FLUSH_DW_USE_GTT);
1418 intel_ring_emit(signaller, upper_32_bits(gtt_offset)); 1407 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1419 intel_ring_emit(signaller, seqno); 1408 intel_ring_emit(signaller, signaller_req->seqno);
1420 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | 1409 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1421 MI_SEMAPHORE_TARGET(waiter->hw_id)); 1410 MI_SEMAPHORE_TARGET(waiter->hw_id));
1422 intel_ring_emit(signaller, 0); 1411 intel_ring_emit(signaller, 0);
@@ -1429,14 +1418,13 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req,
1429 unsigned int num_dwords) 1418 unsigned int num_dwords)
1430{ 1419{
1431 struct intel_engine_cs *signaller = signaller_req->engine; 1420 struct intel_engine_cs *signaller = signaller_req->engine;
1432 struct drm_device *dev = signaller->dev; 1421 struct drm_i915_private *dev_priv = signaller_req->i915;
1433 struct drm_i915_private *dev_priv = dev->dev_private;
1434 struct intel_engine_cs *useless; 1422 struct intel_engine_cs *useless;
1435 enum intel_engine_id id; 1423 enum intel_engine_id id;
1436 int ret, num_rings; 1424 int ret, num_rings;
1437 1425
1438#define MBOX_UPDATE_DWORDS 3 1426#define MBOX_UPDATE_DWORDS 3
1439 num_rings = hweight32(INTEL_INFO(dev)->ring_mask); 1427 num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
1440 num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2); 1428 num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
1441#undef MBOX_UPDATE_DWORDS 1429#undef MBOX_UPDATE_DWORDS
1442 1430
@@ -1448,11 +1436,9 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req,
1448 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id]; 1436 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id];
1449 1437
1450 if (i915_mmio_reg_valid(mbox_reg)) { 1438 if (i915_mmio_reg_valid(mbox_reg)) {
1451 u32 seqno = i915_gem_request_get_seqno(signaller_req);
1452
1453 intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); 1439 intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
1454 intel_ring_emit_reg(signaller, mbox_reg); 1440 intel_ring_emit_reg(signaller, mbox_reg);
1455 intel_ring_emit(signaller, seqno); 1441 intel_ring_emit(signaller, signaller_req->seqno);
1456 } 1442 }
1457 } 1443 }
1458 1444
@@ -1488,17 +1474,45 @@ gen6_add_request(struct drm_i915_gem_request *req)
1488 intel_ring_emit(engine, MI_STORE_DWORD_INDEX); 1474 intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1489 intel_ring_emit(engine, 1475 intel_ring_emit(engine,
1490 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 1476 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1477 intel_ring_emit(engine, req->seqno);
1478 intel_ring_emit(engine, MI_USER_INTERRUPT);
1479 __intel_ring_advance(engine);
1480
1481 return 0;
1482}
1483
1484static int
1485gen8_render_add_request(struct drm_i915_gem_request *req)
1486{
1487 struct intel_engine_cs *engine = req->engine;
1488 int ret;
1489
1490 if (engine->semaphore.signal)
1491 ret = engine->semaphore.signal(req, 8);
1492 else
1493 ret = intel_ring_begin(req, 8);
1494 if (ret)
1495 return ret;
1496
1497 intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6));
1498 intel_ring_emit(engine, (PIPE_CONTROL_GLOBAL_GTT_IVB |
1499 PIPE_CONTROL_CS_STALL |
1500 PIPE_CONTROL_QW_WRITE));
1501 intel_ring_emit(engine, intel_hws_seqno_address(req->engine));
1502 intel_ring_emit(engine, 0);
1491 intel_ring_emit(engine, i915_gem_request_get_seqno(req)); 1503 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1504 /* We're thrashing one dword of HWS. */
1505 intel_ring_emit(engine, 0);
1492 intel_ring_emit(engine, MI_USER_INTERRUPT); 1506 intel_ring_emit(engine, MI_USER_INTERRUPT);
1507 intel_ring_emit(engine, MI_NOOP);
1493 __intel_ring_advance(engine); 1508 __intel_ring_advance(engine);
1494 1509
1495 return 0; 1510 return 0;
1496} 1511}
1497 1512
1498static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, 1513static inline bool i915_gem_has_seqno_wrapped(struct drm_i915_private *dev_priv,
1499 u32 seqno) 1514 u32 seqno)
1500{ 1515{
1501 struct drm_i915_private *dev_priv = dev->dev_private;
1502 return dev_priv->last_seqno < seqno; 1516 return dev_priv->last_seqno < seqno;
1503} 1517}
1504 1518
@@ -1516,7 +1530,9 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
1516 u32 seqno) 1530 u32 seqno)
1517{ 1531{
1518 struct intel_engine_cs *waiter = waiter_req->engine; 1532 struct intel_engine_cs *waiter = waiter_req->engine;
1519 struct drm_i915_private *dev_priv = waiter->dev->dev_private; 1533 struct drm_i915_private *dev_priv = waiter_req->i915;
1534 u64 offset = GEN8_WAIT_OFFSET(waiter, signaller->id);
1535 struct i915_hw_ppgtt *ppgtt;
1520 int ret; 1536 int ret;
1521 1537
1522 ret = intel_ring_begin(waiter_req, 4); 1538 ret = intel_ring_begin(waiter_req, 4);
@@ -1525,14 +1541,20 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
1525 1541
1526 intel_ring_emit(waiter, MI_SEMAPHORE_WAIT | 1542 intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
1527 MI_SEMAPHORE_GLOBAL_GTT | 1543 MI_SEMAPHORE_GLOBAL_GTT |
1528 MI_SEMAPHORE_POLL |
1529 MI_SEMAPHORE_SAD_GTE_SDD); 1544 MI_SEMAPHORE_SAD_GTE_SDD);
1530 intel_ring_emit(waiter, seqno); 1545 intel_ring_emit(waiter, seqno);
1531 intel_ring_emit(waiter, 1546 intel_ring_emit(waiter, lower_32_bits(offset));
1532 lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); 1547 intel_ring_emit(waiter, upper_32_bits(offset));
1533 intel_ring_emit(waiter,
1534 upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1535 intel_ring_advance(waiter); 1548 intel_ring_advance(waiter);
1549
1550 /* When the !RCS engines idle waiting upon a semaphore, they lose their
1551 * pagetables and we must reload them before executing the batch.
1552 * We do this on the i915_switch_context() following the wait and
1553 * before the dispatch.
1554 */
1555 ppgtt = waiter_req->ctx->ppgtt;
1556 if (ppgtt && waiter_req->engine->id != RCS)
1557 ppgtt->pd_dirty_rings |= intel_engine_flag(waiter_req->engine);
1536 return 0; 1558 return 0;
1537} 1559}
1538 1560
@@ -1561,7 +1583,7 @@ gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
1561 return ret; 1583 return ret;
1562 1584
1563 /* If seqno wrap happened, omit the wait with no-ops */ 1585 /* If seqno wrap happened, omit the wait with no-ops */
1564 if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) { 1586 if (likely(!i915_gem_has_seqno_wrapped(waiter_req->i915, seqno))) {
1565 intel_ring_emit(waiter, dw1 | wait_mbox); 1587 intel_ring_emit(waiter, dw1 | wait_mbox);
1566 intel_ring_emit(waiter, seqno); 1588 intel_ring_emit(waiter, seqno);
1567 intel_ring_emit(waiter, 0); 1589 intel_ring_emit(waiter, 0);
@@ -1577,72 +1599,28 @@ gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
1577 return 0; 1599 return 0;
1578} 1600}
1579 1601
1580#define PIPE_CONTROL_FLUSH(ring__, addr__) \ 1602static void
1581do { \ 1603gen5_seqno_barrier(struct intel_engine_cs *ring)
1582 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
1583 PIPE_CONTROL_DEPTH_STALL); \
1584 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
1585 intel_ring_emit(ring__, 0); \
1586 intel_ring_emit(ring__, 0); \
1587} while (0)
1588
1589static int
1590pc_render_add_request(struct drm_i915_gem_request *req)
1591{ 1604{
1592 struct intel_engine_cs *engine = req->engine; 1605 /* MI_STORE are internally buffered by the GPU and not flushed
1593 u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES; 1606 * either by MI_FLUSH or SyncFlush or any other combination of
1594 int ret; 1607 * MI commands.
1595
1596 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
1597 * incoherent with writes to memory, i.e. completely fubar,
1598 * so we need to use PIPE_NOTIFY instead.
1599 * 1608 *
1600 * However, we also need to workaround the qword write 1609 * "Only the submission of the store operation is guaranteed.
1601 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to 1610 * The write result will be complete (coherent) some time later
1602 * memory before requesting an interrupt. 1611 * (this is practically a finite period but there is no guaranteed
1612 * latency)."
1613 *
1614 * Empirically, we observe that we need a delay of at least 75us to
1615 * be sure that the seqno write is visible by the CPU.
1603 */ 1616 */
1604 ret = intel_ring_begin(req, 32); 1617 usleep_range(125, 250);
1605 if (ret)
1606 return ret;
1607
1608 intel_ring_emit(engine,
1609 GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1610 PIPE_CONTROL_WRITE_FLUSH |
1611 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
1612 intel_ring_emit(engine,
1613 engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1614 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1615 intel_ring_emit(engine, 0);
1616 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1617 scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
1618 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1619 scratch_addr += 2 * CACHELINE_BYTES;
1620 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1621 scratch_addr += 2 * CACHELINE_BYTES;
1622 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1623 scratch_addr += 2 * CACHELINE_BYTES;
1624 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1625 scratch_addr += 2 * CACHELINE_BYTES;
1626 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1627
1628 intel_ring_emit(engine,
1629 GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1630 PIPE_CONTROL_WRITE_FLUSH |
1631 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
1632 PIPE_CONTROL_NOTIFY);
1633 intel_ring_emit(engine,
1634 engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1635 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1636 intel_ring_emit(engine, 0);
1637 __intel_ring_advance(engine);
1638
1639 return 0;
1640} 1618}
1641 1619
1642static void 1620static void
1643gen6_seqno_barrier(struct intel_engine_cs *engine) 1621gen6_seqno_barrier(struct intel_engine_cs *engine)
1644{ 1622{
1645 struct drm_i915_private *dev_priv = engine->dev->dev_private; 1623 struct drm_i915_private *dev_priv = engine->i915;
1646 1624
1647 /* Workaround to force correct ordering between irq and seqno writes on 1625 /* Workaround to force correct ordering between irq and seqno writes on
1648 * ivb (and maybe also on snb) by reading from a CS register (like 1626 * ivb (and maybe also on snb) by reading from a CS register (like
@@ -1664,133 +1642,54 @@ gen6_seqno_barrier(struct intel_engine_cs *engine)
1664 spin_unlock_irq(&dev_priv->uncore.lock); 1642 spin_unlock_irq(&dev_priv->uncore.lock);
1665} 1643}
1666 1644
1667static u32
1668ring_get_seqno(struct intel_engine_cs *engine)
1669{
1670 return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
1671}
1672
1673static void 1645static void
1674ring_set_seqno(struct intel_engine_cs *engine, u32 seqno) 1646gen5_irq_enable(struct intel_engine_cs *engine)
1675{
1676 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
1677}
1678
1679static u32
1680pc_render_get_seqno(struct intel_engine_cs *engine)
1681{ 1647{
1682 return engine->scratch.cpu_page[0]; 1648 gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
1683} 1649}
1684 1650
1685static void 1651static void
1686pc_render_set_seqno(struct intel_engine_cs *engine, u32 seqno) 1652gen5_irq_disable(struct intel_engine_cs *engine)
1687{
1688 engine->scratch.cpu_page[0] = seqno;
1689}
1690
1691static bool
1692gen5_ring_get_irq(struct intel_engine_cs *engine)
1693{ 1653{
1694 struct drm_device *dev = engine->dev; 1654 gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
1695 struct drm_i915_private *dev_priv = dev->dev_private;
1696 unsigned long flags;
1697
1698 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1699 return false;
1700
1701 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1702 if (engine->irq_refcount++ == 0)
1703 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1704 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1705
1706 return true;
1707} 1655}
1708 1656
1709static void 1657static void
1710gen5_ring_put_irq(struct intel_engine_cs *engine) 1658i9xx_irq_enable(struct intel_engine_cs *engine)
1711{ 1659{
1712 struct drm_device *dev = engine->dev; 1660 struct drm_i915_private *dev_priv = engine->i915;
1713 struct drm_i915_private *dev_priv = dev->dev_private;
1714 unsigned long flags;
1715
1716 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1717 if (--engine->irq_refcount == 0)
1718 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1719 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1720}
1721
1722static bool
1723i9xx_ring_get_irq(struct intel_engine_cs *engine)
1724{
1725 struct drm_device *dev = engine->dev;
1726 struct drm_i915_private *dev_priv = dev->dev_private;
1727 unsigned long flags;
1728
1729 if (!intel_irqs_enabled(dev_priv))
1730 return false;
1731
1732 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1733 if (engine->irq_refcount++ == 0) {
1734 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1735 I915_WRITE(IMR, dev_priv->irq_mask);
1736 POSTING_READ(IMR);
1737 }
1738 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1739 1661
1740 return true; 1662 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1663 I915_WRITE(IMR, dev_priv->irq_mask);
1664 POSTING_READ_FW(RING_IMR(engine->mmio_base));
1741} 1665}
1742 1666
1743static void 1667static void
1744i9xx_ring_put_irq(struct intel_engine_cs *engine) 1668i9xx_irq_disable(struct intel_engine_cs *engine)
1745{ 1669{
1746 struct drm_device *dev = engine->dev; 1670 struct drm_i915_private *dev_priv = engine->i915;
1747 struct drm_i915_private *dev_priv = dev->dev_private;
1748 unsigned long flags;
1749 1671
1750 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1672 dev_priv->irq_mask |= engine->irq_enable_mask;
1751 if (--engine->irq_refcount == 0) { 1673 I915_WRITE(IMR, dev_priv->irq_mask);
1752 dev_priv->irq_mask |= engine->irq_enable_mask;
1753 I915_WRITE(IMR, dev_priv->irq_mask);
1754 POSTING_READ(IMR);
1755 }
1756 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1757} 1674}
1758 1675
1759static bool 1676static void
1760i8xx_ring_get_irq(struct intel_engine_cs *engine) 1677i8xx_irq_enable(struct intel_engine_cs *engine)
1761{ 1678{
1762 struct drm_device *dev = engine->dev; 1679 struct drm_i915_private *dev_priv = engine->i915;
1763 struct drm_i915_private *dev_priv = dev->dev_private;
1764 unsigned long flags;
1765
1766 if (!intel_irqs_enabled(dev_priv))
1767 return false;
1768
1769 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1770 if (engine->irq_refcount++ == 0) {
1771 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1772 I915_WRITE16(IMR, dev_priv->irq_mask);
1773 POSTING_READ16(IMR);
1774 }
1775 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1776 1680
1777 return true; 1681 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1682 I915_WRITE16(IMR, dev_priv->irq_mask);
1683 POSTING_READ16(RING_IMR(engine->mmio_base));
1778} 1684}
1779 1685
1780static void 1686static void
1781i8xx_ring_put_irq(struct intel_engine_cs *engine) 1687i8xx_irq_disable(struct intel_engine_cs *engine)
1782{ 1688{
1783 struct drm_device *dev = engine->dev; 1689 struct drm_i915_private *dev_priv = engine->i915;
1784 struct drm_i915_private *dev_priv = dev->dev_private;
1785 unsigned long flags;
1786 1690
1787 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1691 dev_priv->irq_mask |= engine->irq_enable_mask;
1788 if (--engine->irq_refcount == 0) { 1692 I915_WRITE16(IMR, dev_priv->irq_mask);
1789 dev_priv->irq_mask |= engine->irq_enable_mask;
1790 I915_WRITE16(IMR, dev_priv->irq_mask);
1791 POSTING_READ16(IMR);
1792 }
1793 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1794} 1693}
1795 1694
1796static int 1695static int
@@ -1824,135 +1723,68 @@ i9xx_add_request(struct drm_i915_gem_request *req)
1824 intel_ring_emit(engine, MI_STORE_DWORD_INDEX); 1723 intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1825 intel_ring_emit(engine, 1724 intel_ring_emit(engine,
1826 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 1725 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1827 intel_ring_emit(engine, i915_gem_request_get_seqno(req)); 1726 intel_ring_emit(engine, req->seqno);
1828 intel_ring_emit(engine, MI_USER_INTERRUPT); 1727 intel_ring_emit(engine, MI_USER_INTERRUPT);
1829 __intel_ring_advance(engine); 1728 __intel_ring_advance(engine);
1830 1729
1831 return 0; 1730 return 0;
1832} 1731}
1833 1732
1834static bool 1733static void
1835gen6_ring_get_irq(struct intel_engine_cs *engine) 1734gen6_irq_enable(struct intel_engine_cs *engine)
1836{ 1735{
1837 struct drm_device *dev = engine->dev; 1736 struct drm_i915_private *dev_priv = engine->i915;
1838 struct drm_i915_private *dev_priv = dev->dev_private;
1839 unsigned long flags;
1840
1841 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1842 return false;
1843
1844 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1845 if (engine->irq_refcount++ == 0) {
1846 if (HAS_L3_DPF(dev) && engine->id == RCS)
1847 I915_WRITE_IMR(engine,
1848 ~(engine->irq_enable_mask |
1849 GT_PARITY_ERROR(dev)));
1850 else
1851 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1852 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1853 }
1854 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1855 1737
1856 return true; 1738 I915_WRITE_IMR(engine,
1739 ~(engine->irq_enable_mask |
1740 engine->irq_keep_mask));
1741 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1857} 1742}
1858 1743
1859static void 1744static void
1860gen6_ring_put_irq(struct intel_engine_cs *engine) 1745gen6_irq_disable(struct intel_engine_cs *engine)
1861{ 1746{
1862 struct drm_device *dev = engine->dev; 1747 struct drm_i915_private *dev_priv = engine->i915;
1863 struct drm_i915_private *dev_priv = dev->dev_private;
1864 unsigned long flags;
1865 1748
1866 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1749 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1867 if (--engine->irq_refcount == 0) { 1750 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1868 if (HAS_L3_DPF(dev) && engine->id == RCS)
1869 I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev));
1870 else
1871 I915_WRITE_IMR(engine, ~0);
1872 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1873 }
1874 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1875} 1751}
1876 1752
1877static bool 1753static void
1878hsw_vebox_get_irq(struct intel_engine_cs *engine) 1754hsw_vebox_irq_enable(struct intel_engine_cs *engine)
1879{ 1755{
1880 struct drm_device *dev = engine->dev; 1756 struct drm_i915_private *dev_priv = engine->i915;
1881 struct drm_i915_private *dev_priv = dev->dev_private;
1882 unsigned long flags;
1883
1884 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1885 return false;
1886 1757
1887 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1758 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1888 if (engine->irq_refcount++ == 0) { 1759 gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
1889 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1890 gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
1891 }
1892 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1893
1894 return true;
1895} 1760}
1896 1761
1897static void 1762static void
1898hsw_vebox_put_irq(struct intel_engine_cs *engine) 1763hsw_vebox_irq_disable(struct intel_engine_cs *engine)
1899{ 1764{
1900 struct drm_device *dev = engine->dev; 1765 struct drm_i915_private *dev_priv = engine->i915;
1901 struct drm_i915_private *dev_priv = dev->dev_private;
1902 unsigned long flags;
1903 1766
1904 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1767 I915_WRITE_IMR(engine, ~0);
1905 if (--engine->irq_refcount == 0) { 1768 gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
1906 I915_WRITE_IMR(engine, ~0);
1907 gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
1908 }
1909 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1910} 1769}
1911 1770
1912static bool 1771static void
1913gen8_ring_get_irq(struct intel_engine_cs *engine) 1772gen8_irq_enable(struct intel_engine_cs *engine)
1914{ 1773{
1915 struct drm_device *dev = engine->dev; 1774 struct drm_i915_private *dev_priv = engine->i915;
1916 struct drm_i915_private *dev_priv = dev->dev_private;
1917 unsigned long flags;
1918
1919 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1920 return false;
1921 1775
1922 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1776 I915_WRITE_IMR(engine,
1923 if (engine->irq_refcount++ == 0) { 1777 ~(engine->irq_enable_mask |
1924 if (HAS_L3_DPF(dev) && engine->id == RCS) { 1778 engine->irq_keep_mask));
1925 I915_WRITE_IMR(engine, 1779 POSTING_READ_FW(RING_IMR(engine->mmio_base));
1926 ~(engine->irq_enable_mask |
1927 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1928 } else {
1929 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1930 }
1931 POSTING_READ(RING_IMR(engine->mmio_base));
1932 }
1933 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1934
1935 return true;
1936} 1780}
1937 1781
1938static void 1782static void
1939gen8_ring_put_irq(struct intel_engine_cs *engine) 1783gen8_irq_disable(struct intel_engine_cs *engine)
1940{ 1784{
1941 struct drm_device *dev = engine->dev; 1785 struct drm_i915_private *dev_priv = engine->i915;
1942 struct drm_i915_private *dev_priv = dev->dev_private;
1943 unsigned long flags;
1944 1786
1945 spin_lock_irqsave(&dev_priv->irq_lock, flags); 1787 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1946 if (--engine->irq_refcount == 0) {
1947 if (HAS_L3_DPF(dev) && engine->id == RCS) {
1948 I915_WRITE_IMR(engine,
1949 ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1950 } else {
1951 I915_WRITE_IMR(engine, ~0);
1952 }
1953 POSTING_READ(RING_IMR(engine->mmio_base));
1954 }
1955 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1956} 1788}
1957 1789
1958static int 1790static int
@@ -2066,12 +1898,12 @@ i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
2066 1898
2067static void cleanup_phys_status_page(struct intel_engine_cs *engine) 1899static void cleanup_phys_status_page(struct intel_engine_cs *engine)
2068{ 1900{
2069 struct drm_i915_private *dev_priv = to_i915(engine->dev); 1901 struct drm_i915_private *dev_priv = engine->i915;
2070 1902
2071 if (!dev_priv->status_page_dmah) 1903 if (!dev_priv->status_page_dmah)
2072 return; 1904 return;
2073 1905
2074 drm_pci_free(engine->dev, dev_priv->status_page_dmah); 1906 drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
2075 engine->status_page.page_addr = NULL; 1907 engine->status_page.page_addr = NULL;
2076} 1908}
2077 1909
@@ -2097,10 +1929,10 @@ static int init_status_page(struct intel_engine_cs *engine)
2097 unsigned flags; 1929 unsigned flags;
2098 int ret; 1930 int ret;
2099 1931
2100 obj = i915_gem_alloc_object(engine->dev, 4096); 1932 obj = i915_gem_object_create(&engine->i915->drm, 4096);
2101 if (obj == NULL) { 1933 if (IS_ERR(obj)) {
2102 DRM_ERROR("Failed to allocate status page\n"); 1934 DRM_ERROR("Failed to allocate status page\n");
2103 return -ENOMEM; 1935 return PTR_ERR(obj);
2104 } 1936 }
2105 1937
2106 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); 1938 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
@@ -2108,7 +1940,7 @@ static int init_status_page(struct intel_engine_cs *engine)
2108 goto err_unref; 1940 goto err_unref;
2109 1941
2110 flags = 0; 1942 flags = 0;
2111 if (!HAS_LLC(engine->dev)) 1943 if (!HAS_LLC(engine->i915))
2112 /* On g33, we cannot place HWS above 256MiB, so 1944 /* On g33, we cannot place HWS above 256MiB, so
2113 * restrict its pinning to the low mappable arena. 1945 * restrict its pinning to the low mappable arena.
2114 * Though this restriction is not documented for 1946 * Though this restriction is not documented for
@@ -2142,11 +1974,11 @@ err_unref:
2142 1974
2143static int init_phys_status_page(struct intel_engine_cs *engine) 1975static int init_phys_status_page(struct intel_engine_cs *engine)
2144{ 1976{
2145 struct drm_i915_private *dev_priv = engine->dev->dev_private; 1977 struct drm_i915_private *dev_priv = engine->i915;
2146 1978
2147 if (!dev_priv->status_page_dmah) { 1979 if (!dev_priv->status_page_dmah) {
2148 dev_priv->status_page_dmah = 1980 dev_priv->status_page_dmah =
2149 drm_pci_alloc(engine->dev, PAGE_SIZE, PAGE_SIZE); 1981 drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
2150 if (!dev_priv->status_page_dmah) 1982 if (!dev_priv->status_page_dmah)
2151 return -ENOMEM; 1983 return -ENOMEM;
2152 } 1984 }
@@ -2159,20 +1991,22 @@ static int init_phys_status_page(struct intel_engine_cs *engine)
2159 1991
2160void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf) 1992void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2161{ 1993{
1994 GEM_BUG_ON(ringbuf->vma == NULL);
1995 GEM_BUG_ON(ringbuf->virtual_start == NULL);
1996
2162 if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen) 1997 if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
2163 i915_gem_object_unpin_map(ringbuf->obj); 1998 i915_gem_object_unpin_map(ringbuf->obj);
2164 else 1999 else
2165 iounmap(ringbuf->virtual_start); 2000 i915_vma_unpin_iomap(ringbuf->vma);
2166 ringbuf->virtual_start = NULL; 2001 ringbuf->virtual_start = NULL;
2167 ringbuf->vma = NULL; 2002
2168 i915_gem_object_ggtt_unpin(ringbuf->obj); 2003 i915_gem_object_ggtt_unpin(ringbuf->obj);
2004 ringbuf->vma = NULL;
2169} 2005}
2170 2006
2171int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev, 2007int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv,
2172 struct intel_ringbuffer *ringbuf) 2008 struct intel_ringbuffer *ringbuf)
2173{ 2009{
2174 struct drm_i915_private *dev_priv = to_i915(dev);
2175 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2176 struct drm_i915_gem_object *obj = ringbuf->obj; 2010 struct drm_i915_gem_object *obj = ringbuf->obj;
2177 /* Ring wraparound at offset 0 sometimes hangs. No idea why. */ 2011 /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
2178 unsigned flags = PIN_OFFSET_BIAS | 4096; 2012 unsigned flags = PIN_OFFSET_BIAS | 4096;
@@ -2206,10 +2040,9 @@ int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
2206 /* Access through the GTT requires the device to be awake. */ 2040 /* Access through the GTT requires the device to be awake. */
2207 assert_rpm_wakelock_held(dev_priv); 2041 assert_rpm_wakelock_held(dev_priv);
2208 2042
2209 addr = ioremap_wc(ggtt->mappable_base + 2043 addr = i915_vma_pin_iomap(i915_gem_obj_to_ggtt(obj));
2210 i915_gem_obj_ggtt_offset(obj), ringbuf->size); 2044 if (IS_ERR(addr)) {
2211 if (addr == NULL) { 2045 ret = PTR_ERR(addr);
2212 ret = -ENOMEM;
2213 goto err_unpin; 2046 goto err_unpin;
2214 } 2047 }
2215 } 2048 }
@@ -2238,9 +2071,9 @@ static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
2238 if (!HAS_LLC(dev)) 2071 if (!HAS_LLC(dev))
2239 obj = i915_gem_object_create_stolen(dev, ringbuf->size); 2072 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
2240 if (obj == NULL) 2073 if (obj == NULL)
2241 obj = i915_gem_alloc_object(dev, ringbuf->size); 2074 obj = i915_gem_object_create(dev, ringbuf->size);
2242 if (obj == NULL) 2075 if (IS_ERR(obj))
2243 return -ENOMEM; 2076 return PTR_ERR(obj);
2244 2077
2245 /* mark ring buffers as read-only from GPU side by default */ 2078 /* mark ring buffers as read-only from GPU side by default */
2246 obj->gt_ro = 1; 2079 obj->gt_ro = 1;
@@ -2272,13 +2105,13 @@ intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
2272 * of the buffer. 2105 * of the buffer.
2273 */ 2106 */
2274 ring->effective_size = size; 2107 ring->effective_size = size;
2275 if (IS_I830(engine->dev) || IS_845G(engine->dev)) 2108 if (IS_I830(engine->i915) || IS_845G(engine->i915))
2276 ring->effective_size -= 2 * CACHELINE_BYTES; 2109 ring->effective_size -= 2 * CACHELINE_BYTES;
2277 2110
2278 ring->last_retired_head = -1; 2111 ring->last_retired_head = -1;
2279 intel_ring_update_space(ring); 2112 intel_ring_update_space(ring);
2280 2113
2281 ret = intel_alloc_ringbuffer_obj(engine->dev, ring); 2114 ret = intel_alloc_ringbuffer_obj(&engine->i915->drm, ring);
2282 if (ret) { 2115 if (ret) {
2283 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n", 2116 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
2284 engine->name, ret); 2117 engine->name, ret);
@@ -2298,15 +2131,67 @@ intel_ringbuffer_free(struct intel_ringbuffer *ring)
2298 kfree(ring); 2131 kfree(ring);
2299} 2132}
2300 2133
2134static int intel_ring_context_pin(struct i915_gem_context *ctx,
2135 struct intel_engine_cs *engine)
2136{
2137 struct intel_context *ce = &ctx->engine[engine->id];
2138 int ret;
2139
2140 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
2141
2142 if (ce->pin_count++)
2143 return 0;
2144
2145 if (ce->state) {
2146 ret = i915_gem_obj_ggtt_pin(ce->state, ctx->ggtt_alignment, 0);
2147 if (ret)
2148 goto error;
2149 }
2150
2151 /* The kernel context is only used as a placeholder for flushing the
2152 * active context. It is never used for submitting user rendering and
2153 * as such never requires the golden render context, and so we can skip
2154 * emitting it when we switch to the kernel context. This is required
2155 * as during eviction we cannot allocate and pin the renderstate in
2156 * order to initialise the context.
2157 */
2158 if (ctx == ctx->i915->kernel_context)
2159 ce->initialised = true;
2160
2161 i915_gem_context_reference(ctx);
2162 return 0;
2163
2164error:
2165 ce->pin_count = 0;
2166 return ret;
2167}
2168
2169static void intel_ring_context_unpin(struct i915_gem_context *ctx,
2170 struct intel_engine_cs *engine)
2171{
2172 struct intel_context *ce = &ctx->engine[engine->id];
2173
2174 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
2175
2176 if (--ce->pin_count)
2177 return;
2178
2179 if (ce->state)
2180 i915_gem_object_ggtt_unpin(ce->state);
2181
2182 i915_gem_context_unreference(ctx);
2183}
2184
2301static int intel_init_ring_buffer(struct drm_device *dev, 2185static int intel_init_ring_buffer(struct drm_device *dev,
2302 struct intel_engine_cs *engine) 2186 struct intel_engine_cs *engine)
2303{ 2187{
2188 struct drm_i915_private *dev_priv = to_i915(dev);
2304 struct intel_ringbuffer *ringbuf; 2189 struct intel_ringbuffer *ringbuf;
2305 int ret; 2190 int ret;
2306 2191
2307 WARN_ON(engine->buffer); 2192 WARN_ON(engine->buffer);
2308 2193
2309 engine->dev = dev; 2194 engine->i915 = dev_priv;
2310 INIT_LIST_HEAD(&engine->active_list); 2195 INIT_LIST_HEAD(&engine->active_list);
2311 INIT_LIST_HEAD(&engine->request_list); 2196 INIT_LIST_HEAD(&engine->request_list);
2312 INIT_LIST_HEAD(&engine->execlist_queue); 2197 INIT_LIST_HEAD(&engine->execlist_queue);
@@ -2315,7 +2200,20 @@ static int intel_init_ring_buffer(struct drm_device *dev,
2315 memset(engine->semaphore.sync_seqno, 0, 2200 memset(engine->semaphore.sync_seqno, 0,
2316 sizeof(engine->semaphore.sync_seqno)); 2201 sizeof(engine->semaphore.sync_seqno));
2317 2202
2318 init_waitqueue_head(&engine->irq_queue); 2203 ret = intel_engine_init_breadcrumbs(engine);
2204 if (ret)
2205 goto error;
2206
2207 /* We may need to do things with the shrinker which
2208 * require us to immediately switch back to the default
2209 * context. This can cause a problem as pinning the
2210 * default context also requires GTT space which may not
2211 * be available. To avoid this we always pin the default
2212 * context.
2213 */
2214 ret = intel_ring_context_pin(dev_priv->kernel_context, engine);
2215 if (ret)
2216 goto error;
2319 2217
2320 ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE); 2218 ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE);
2321 if (IS_ERR(ringbuf)) { 2219 if (IS_ERR(ringbuf)) {
@@ -2324,7 +2222,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
2324 } 2222 }
2325 engine->buffer = ringbuf; 2223 engine->buffer = ringbuf;
2326 2224
2327 if (I915_NEED_GFX_HWS(dev)) { 2225 if (I915_NEED_GFX_HWS(dev_priv)) {
2328 ret = init_status_page(engine); 2226 ret = init_status_page(engine);
2329 if (ret) 2227 if (ret)
2330 goto error; 2228 goto error;
@@ -2335,7 +2233,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
2335 goto error; 2233 goto error;
2336 } 2234 }
2337 2235
2338 ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf); 2236 ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ringbuf);
2339 if (ret) { 2237 if (ret) {
2340 DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n", 2238 DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
2341 engine->name, ret); 2239 engine->name, ret);
@@ -2361,11 +2259,11 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
2361 if (!intel_engine_initialized(engine)) 2259 if (!intel_engine_initialized(engine))
2362 return; 2260 return;
2363 2261
2364 dev_priv = to_i915(engine->dev); 2262 dev_priv = engine->i915;
2365 2263
2366 if (engine->buffer) { 2264 if (engine->buffer) {
2367 intel_stop_engine(engine); 2265 intel_stop_engine(engine);
2368 WARN_ON(!IS_GEN2(engine->dev) && (I915_READ_MODE(engine) & MODE_IDLE) == 0); 2266 WARN_ON(!IS_GEN2(dev_priv) && (I915_READ_MODE(engine) & MODE_IDLE) == 0);
2369 2267
2370 intel_unpin_ringbuffer_obj(engine->buffer); 2268 intel_unpin_ringbuffer_obj(engine->buffer);
2371 intel_ringbuffer_free(engine->buffer); 2269 intel_ringbuffer_free(engine->buffer);
@@ -2375,7 +2273,7 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
2375 if (engine->cleanup) 2273 if (engine->cleanup)
2376 engine->cleanup(engine); 2274 engine->cleanup(engine);
2377 2275
2378 if (I915_NEED_GFX_HWS(engine->dev)) { 2276 if (I915_NEED_GFX_HWS(dev_priv)) {
2379 cleanup_status_page(engine); 2277 cleanup_status_page(engine);
2380 } else { 2278 } else {
2381 WARN_ON(engine->id != RCS); 2279 WARN_ON(engine->id != RCS);
@@ -2384,7 +2282,11 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
2384 2282
2385 i915_cmd_parser_fini_ring(engine); 2283 i915_cmd_parser_fini_ring(engine);
2386 i915_gem_batch_pool_fini(&engine->batch_pool); 2284 i915_gem_batch_pool_fini(&engine->batch_pool);
2387 engine->dev = NULL; 2285 intel_engine_fini_breadcrumbs(engine);
2286
2287 intel_ring_context_unpin(dev_priv->kernel_context, engine);
2288
2289 engine->i915 = NULL;
2388} 2290}
2389 2291
2390int intel_engine_idle(struct intel_engine_cs *engine) 2292int intel_engine_idle(struct intel_engine_cs *engine)
@@ -2407,46 +2309,22 @@ int intel_engine_idle(struct intel_engine_cs *engine)
2407 2309
2408int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request) 2310int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
2409{ 2311{
2410 request->ringbuf = request->engine->buffer; 2312 int ret;
2411 return 0;
2412}
2413 2313
2414int intel_ring_reserve_space(struct drm_i915_gem_request *request) 2314 /* Flush enough space to reduce the likelihood of waiting after
2415{ 2315 * we start building the request - in which case we will just
2416 /* 2316 * have to repeat work.
2417 * The first call merely notes the reserve request and is common for
2418 * all back ends. The subsequent localised _begin() call actually
2419 * ensures that the reservation is available. Without the begin, if
2420 * the request creator immediately submitted the request without
2421 * adding any commands to it then there might not actually be
2422 * sufficient room for the submission commands.
2423 */ 2317 */
2424 intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST); 2318 request->reserved_space += LEGACY_REQUEST_SIZE;
2425
2426 return intel_ring_begin(request, 0);
2427}
2428 2319
2429void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size) 2320 request->ringbuf = request->engine->buffer;
2430{
2431 GEM_BUG_ON(ringbuf->reserved_size);
2432 ringbuf->reserved_size = size;
2433}
2434
2435void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
2436{
2437 GEM_BUG_ON(!ringbuf->reserved_size);
2438 ringbuf->reserved_size = 0;
2439}
2440 2321
2441void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf) 2322 ret = intel_ring_begin(request, 0);
2442{ 2323 if (ret)
2443 GEM_BUG_ON(!ringbuf->reserved_size); 2324 return ret;
2444 ringbuf->reserved_size = 0;
2445}
2446 2325
2447void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf) 2326 request->reserved_space -= LEGACY_REQUEST_SIZE;
2448{ 2327 return 0;
2449 GEM_BUG_ON(ringbuf->reserved_size);
2450} 2328}
2451 2329
2452static int wait_for_space(struct drm_i915_gem_request *req, int bytes) 2330static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
@@ -2468,7 +2346,7 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
2468 * 2346 *
2469 * See also i915_gem_request_alloc() and i915_add_request(). 2347 * See also i915_gem_request_alloc() and i915_add_request().
2470 */ 2348 */
2471 GEM_BUG_ON(!ringbuf->reserved_size); 2349 GEM_BUG_ON(!req->reserved_space);
2472 2350
2473 list_for_each_entry(target, &engine->request_list, list) { 2351 list_for_each_entry(target, &engine->request_list, list) {
2474 unsigned space; 2352 unsigned space;
@@ -2503,7 +2381,7 @@ int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
2503 int total_bytes, wait_bytes; 2381 int total_bytes, wait_bytes;
2504 bool need_wrap = false; 2382 bool need_wrap = false;
2505 2383
2506 total_bytes = bytes + ringbuf->reserved_size; 2384 total_bytes = bytes + req->reserved_space;
2507 2385
2508 if (unlikely(bytes > remain_usable)) { 2386 if (unlikely(bytes > remain_usable)) {
2509 /* 2387 /*
@@ -2519,7 +2397,7 @@ int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
2519 * and only need to effectively wait for the reserved 2397 * and only need to effectively wait for the reserved
2520 * size space from the start of ringbuffer. 2398 * size space from the start of ringbuffer.
2521 */ 2399 */
2522 wait_bytes = remain_actual + ringbuf->reserved_size; 2400 wait_bytes = remain_actual + req->reserved_space;
2523 } else { 2401 } else {
2524 /* No wrapping required, just waiting. */ 2402 /* No wrapping required, just waiting. */
2525 wait_bytes = total_bytes; 2403 wait_bytes = total_bytes;
@@ -2576,7 +2454,7 @@ int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
2576 2454
2577void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno) 2455void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
2578{ 2456{
2579 struct drm_i915_private *dev_priv = to_i915(engine->dev); 2457 struct drm_i915_private *dev_priv = engine->i915;
2580 2458
2581 /* Our semaphore implementation is strictly monotonic (i.e. we proceed 2459 /* Our semaphore implementation is strictly monotonic (i.e. we proceed
2582 * so long as the semaphore value in the register/page is greater 2460 * so long as the semaphore value in the register/page is greater
@@ -2586,7 +2464,7 @@ void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
2586 * the semaphore value, then when the seqno moves backwards all 2464 * the semaphore value, then when the seqno moves backwards all
2587 * future waits will complete instantly (causing rendering corruption). 2465 * future waits will complete instantly (causing rendering corruption).
2588 */ 2466 */
2589 if (INTEL_INFO(dev_priv)->gen == 6 || INTEL_INFO(dev_priv)->gen == 7) { 2467 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
2590 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0); 2468 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
2591 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0); 2469 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
2592 if (HAS_VEBOX(dev_priv)) 2470 if (HAS_VEBOX(dev_priv))
@@ -2603,43 +2481,58 @@ void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
2603 memset(engine->semaphore.sync_seqno, 0, 2481 memset(engine->semaphore.sync_seqno, 0,
2604 sizeof(engine->semaphore.sync_seqno)); 2482 sizeof(engine->semaphore.sync_seqno));
2605 2483
2606 engine->set_seqno(engine, seqno); 2484 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
2485 if (engine->irq_seqno_barrier)
2486 engine->irq_seqno_barrier(engine);
2607 engine->last_submitted_seqno = seqno; 2487 engine->last_submitted_seqno = seqno;
2608 2488
2609 engine->hangcheck.seqno = seqno; 2489 engine->hangcheck.seqno = seqno;
2490
2491 /* After manually advancing the seqno, fake the interrupt in case
2492 * there are any waiters for that seqno.
2493 */
2494 rcu_read_lock();
2495 intel_engine_wakeup(engine);
2496 rcu_read_unlock();
2610} 2497}
2611 2498
2612static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine, 2499static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
2613 u32 value) 2500 u32 value)
2614{ 2501{
2615 struct drm_i915_private *dev_priv = engine->dev->dev_private; 2502 struct drm_i915_private *dev_priv = engine->i915;
2503
2504 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
2616 2505
2617 /* Every tail move must follow the sequence below */ 2506 /* Every tail move must follow the sequence below */
2618 2507
2619 /* Disable notification that the ring is IDLE. The GT 2508 /* Disable notification that the ring is IDLE. The GT
2620 * will then assume that it is busy and bring it out of rc6. 2509 * will then assume that it is busy and bring it out of rc6.
2621 */ 2510 */
2622 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 2511 I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
2623 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 2512 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2624 2513
2625 /* Clear the context id. Here be magic! */ 2514 /* Clear the context id. Here be magic! */
2626 I915_WRITE64(GEN6_BSD_RNCID, 0x0); 2515 I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
2627 2516
2628 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 2517 /* Wait for the ring not to be idle, i.e. for it to wake up. */
2629 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) & 2518 if (intel_wait_for_register_fw(dev_priv,
2630 GEN6_BSD_SLEEP_INDICATOR) == 0, 2519 GEN6_BSD_SLEEP_PSMI_CONTROL,
2631 50)) 2520 GEN6_BSD_SLEEP_INDICATOR,
2521 0,
2522 50))
2632 DRM_ERROR("timed out waiting for the BSD ring to wake up\n"); 2523 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2633 2524
2634 /* Now that the ring is fully powered up, update the tail */ 2525 /* Now that the ring is fully powered up, update the tail */
2635 I915_WRITE_TAIL(engine, value); 2526 I915_WRITE_FW(RING_TAIL(engine->mmio_base), value);
2636 POSTING_READ(RING_TAIL(engine->mmio_base)); 2527 POSTING_READ_FW(RING_TAIL(engine->mmio_base));
2637 2528
2638 /* Let the ring send IDLE messages to the GT again, 2529 /* Let the ring send IDLE messages to the GT again,
2639 * and so let it sleep to conserve power when idle. 2530 * and so let it sleep to conserve power when idle.
2640 */ 2531 */
2641 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 2532 I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
2642 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 2533 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2534
2535 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2643} 2536}
2644 2537
2645static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, 2538static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
@@ -2654,7 +2547,7 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
2654 return ret; 2547 return ret;
2655 2548
2656 cmd = MI_FLUSH_DW; 2549 cmd = MI_FLUSH_DW;
2657 if (INTEL_INFO(engine->dev)->gen >= 8) 2550 if (INTEL_GEN(req->i915) >= 8)
2658 cmd += 1; 2551 cmd += 1;
2659 2552
2660 /* We always require a command barrier so that subsequent 2553 /* We always require a command barrier so that subsequent
@@ -2676,7 +2569,7 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
2676 intel_ring_emit(engine, cmd); 2569 intel_ring_emit(engine, cmd);
2677 intel_ring_emit(engine, 2570 intel_ring_emit(engine,
2678 I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); 2571 I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2679 if (INTEL_INFO(engine->dev)->gen >= 8) { 2572 if (INTEL_GEN(req->i915) >= 8) {
2680 intel_ring_emit(engine, 0); /* upper addr */ 2573 intel_ring_emit(engine, 0); /* upper addr */
2681 intel_ring_emit(engine, 0); /* value */ 2574 intel_ring_emit(engine, 0); /* value */
2682 } else { 2575 } else {
@@ -2767,7 +2660,6 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
2767 u32 invalidate, u32 flush) 2660 u32 invalidate, u32 flush)
2768{ 2661{
2769 struct intel_engine_cs *engine = req->engine; 2662 struct intel_engine_cs *engine = req->engine;
2770 struct drm_device *dev = engine->dev;
2771 uint32_t cmd; 2663 uint32_t cmd;
2772 int ret; 2664 int ret;
2773 2665
@@ -2776,7 +2668,7 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
2776 return ret; 2668 return ret;
2777 2669
2778 cmd = MI_FLUSH_DW; 2670 cmd = MI_FLUSH_DW;
2779 if (INTEL_INFO(dev)->gen >= 8) 2671 if (INTEL_GEN(req->i915) >= 8)
2780 cmd += 1; 2672 cmd += 1;
2781 2673
2782 /* We always require a command barrier so that subsequent 2674 /* We always require a command barrier so that subsequent
@@ -2797,7 +2689,7 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
2797 intel_ring_emit(engine, cmd); 2689 intel_ring_emit(engine, cmd);
2798 intel_ring_emit(engine, 2690 intel_ring_emit(engine,
2799 I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); 2691 I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2800 if (INTEL_INFO(dev)->gen >= 8) { 2692 if (INTEL_GEN(req->i915) >= 8) {
2801 intel_ring_emit(engine, 0); /* upper addr */ 2693 intel_ring_emit(engine, 0); /* upper addr */
2802 intel_ring_emit(engine, 0); /* value */ 2694 intel_ring_emit(engine, 0); /* value */
2803 } else { 2695 } else {
@@ -2809,11 +2701,159 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
2809 return 0; 2701 return 0;
2810} 2702}
2811 2703
2704static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
2705 struct intel_engine_cs *engine)
2706{
2707 struct drm_i915_gem_object *obj;
2708 int ret, i;
2709
2710 if (!i915_semaphore_is_enabled(dev_priv))
2711 return;
2712
2713 if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore_obj) {
2714 obj = i915_gem_object_create(&dev_priv->drm, 4096);
2715 if (IS_ERR(obj)) {
2716 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2717 i915.semaphores = 0;
2718 } else {
2719 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2720 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2721 if (ret != 0) {
2722 drm_gem_object_unreference(&obj->base);
2723 DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2724 i915.semaphores = 0;
2725 } else {
2726 dev_priv->semaphore_obj = obj;
2727 }
2728 }
2729 }
2730
2731 if (!i915_semaphore_is_enabled(dev_priv))
2732 return;
2733
2734 if (INTEL_GEN(dev_priv) >= 8) {
2735 u64 offset = i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj);
2736
2737 engine->semaphore.sync_to = gen8_ring_sync;
2738 engine->semaphore.signal = gen8_xcs_signal;
2739
2740 for (i = 0; i < I915_NUM_ENGINES; i++) {
2741 u64 ring_offset;
2742
2743 if (i != engine->id)
2744 ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
2745 else
2746 ring_offset = MI_SEMAPHORE_SYNC_INVALID;
2747
2748 engine->semaphore.signal_ggtt[i] = ring_offset;
2749 }
2750 } else if (INTEL_GEN(dev_priv) >= 6) {
2751 engine->semaphore.sync_to = gen6_ring_sync;
2752 engine->semaphore.signal = gen6_signal;
2753
2754 /*
2755 * The current semaphore is only applied on pre-gen8
2756 * platform. And there is no VCS2 ring on the pre-gen8
2757 * platform. So the semaphore between RCS and VCS2 is
2758 * initialized as INVALID. Gen8 will initialize the
2759 * sema between VCS2 and RCS later.
2760 */
2761 for (i = 0; i < I915_NUM_ENGINES; i++) {
2762 static const struct {
2763 u32 wait_mbox;
2764 i915_reg_t mbox_reg;
2765 } sem_data[I915_NUM_ENGINES][I915_NUM_ENGINES] = {
2766 [RCS] = {
2767 [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RV, .mbox_reg = GEN6_VRSYNC },
2768 [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RB, .mbox_reg = GEN6_BRSYNC },
2769 [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
2770 },
2771 [VCS] = {
2772 [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VR, .mbox_reg = GEN6_RVSYNC },
2773 [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VB, .mbox_reg = GEN6_BVSYNC },
2774 [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
2775 },
2776 [BCS] = {
2777 [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BR, .mbox_reg = GEN6_RBSYNC },
2778 [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BV, .mbox_reg = GEN6_VBSYNC },
2779 [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
2780 },
2781 [VECS] = {
2782 [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
2783 [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
2784 [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
2785 },
2786 };
2787 u32 wait_mbox;
2788 i915_reg_t mbox_reg;
2789
2790 if (i == engine->id || i == VCS2) {
2791 wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
2792 mbox_reg = GEN6_NOSYNC;
2793 } else {
2794 wait_mbox = sem_data[engine->id][i].wait_mbox;
2795 mbox_reg = sem_data[engine->id][i].mbox_reg;
2796 }
2797
2798 engine->semaphore.mbox.wait[i] = wait_mbox;
2799 engine->semaphore.mbox.signal[i] = mbox_reg;
2800 }
2801 }
2802}
2803
2804static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
2805 struct intel_engine_cs *engine)
2806{
2807 if (INTEL_GEN(dev_priv) >= 8) {
2808 engine->irq_enable = gen8_irq_enable;
2809 engine->irq_disable = gen8_irq_disable;
2810 engine->irq_seqno_barrier = gen6_seqno_barrier;
2811 } else if (INTEL_GEN(dev_priv) >= 6) {
2812 engine->irq_enable = gen6_irq_enable;
2813 engine->irq_disable = gen6_irq_disable;
2814 engine->irq_seqno_barrier = gen6_seqno_barrier;
2815 } else if (INTEL_GEN(dev_priv) >= 5) {
2816 engine->irq_enable = gen5_irq_enable;
2817 engine->irq_disable = gen5_irq_disable;
2818 engine->irq_seqno_barrier = gen5_seqno_barrier;
2819 } else if (INTEL_GEN(dev_priv) >= 3) {
2820 engine->irq_enable = i9xx_irq_enable;
2821 engine->irq_disable = i9xx_irq_disable;
2822 } else {
2823 engine->irq_enable = i8xx_irq_enable;
2824 engine->irq_disable = i8xx_irq_disable;
2825 }
2826}
2827
2828static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
2829 struct intel_engine_cs *engine)
2830{
2831 engine->init_hw = init_ring_common;
2832 engine->write_tail = ring_write_tail;
2833
2834 engine->add_request = i9xx_add_request;
2835 if (INTEL_GEN(dev_priv) >= 6)
2836 engine->add_request = gen6_add_request;
2837
2838 if (INTEL_GEN(dev_priv) >= 8)
2839 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2840 else if (INTEL_GEN(dev_priv) >= 6)
2841 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2842 else if (INTEL_GEN(dev_priv) >= 4)
2843 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
2844 else if (IS_I830(dev_priv) || IS_845G(dev_priv))
2845 engine->dispatch_execbuffer = i830_dispatch_execbuffer;
2846 else
2847 engine->dispatch_execbuffer = i915_dispatch_execbuffer;
2848
2849 intel_ring_init_irq(dev_priv, engine);
2850 intel_ring_init_semaphores(dev_priv, engine);
2851}
2852
2812int intel_init_render_ring_buffer(struct drm_device *dev) 2853int intel_init_render_ring_buffer(struct drm_device *dev)
2813{ 2854{
2814 struct drm_i915_private *dev_priv = dev->dev_private; 2855 struct drm_i915_private *dev_priv = to_i915(dev);
2815 struct intel_engine_cs *engine = &dev_priv->engine[RCS]; 2856 struct intel_engine_cs *engine = &dev_priv->engine[RCS];
2816 struct drm_i915_gem_object *obj;
2817 int ret; 2857 int ret;
2818 2858
2819 engine->name = "render ring"; 2859 engine->name = "render ring";
@@ -2822,140 +2862,49 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
2822 engine->hw_id = 0; 2862 engine->hw_id = 0;
2823 engine->mmio_base = RENDER_RING_BASE; 2863 engine->mmio_base = RENDER_RING_BASE;
2824 2864
2825 if (INTEL_INFO(dev)->gen >= 8) { 2865 intel_ring_default_vfuncs(dev_priv, engine);
2826 if (i915_semaphore_is_enabled(dev)) {
2827 obj = i915_gem_alloc_object(dev, 4096);
2828 if (obj == NULL) {
2829 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2830 i915.semaphores = 0;
2831 } else {
2832 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2833 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2834 if (ret != 0) {
2835 drm_gem_object_unreference(&obj->base);
2836 DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2837 i915.semaphores = 0;
2838 } else
2839 dev_priv->semaphore_obj = obj;
2840 }
2841 }
2842 2866
2867 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2868 if (HAS_L3_DPF(dev_priv))
2869 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2870
2871 if (INTEL_GEN(dev_priv) >= 8) {
2843 engine->init_context = intel_rcs_ctx_init; 2872 engine->init_context = intel_rcs_ctx_init;
2844 engine->add_request = gen6_add_request; 2873 engine->add_request = gen8_render_add_request;
2845 engine->flush = gen8_render_ring_flush; 2874 engine->flush = gen8_render_ring_flush;
2846 engine->irq_get = gen8_ring_get_irq; 2875 if (i915_semaphore_is_enabled(dev_priv))
2847 engine->irq_put = gen8_ring_put_irq;
2848 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2849 engine->irq_seqno_barrier = gen6_seqno_barrier;
2850 engine->get_seqno = ring_get_seqno;
2851 engine->set_seqno = ring_set_seqno;
2852 if (i915_semaphore_is_enabled(dev)) {
2853 WARN_ON(!dev_priv->semaphore_obj);
2854 engine->semaphore.sync_to = gen8_ring_sync;
2855 engine->semaphore.signal = gen8_rcs_signal; 2876 engine->semaphore.signal = gen8_rcs_signal;
2856 GEN8_RING_SEMAPHORE_INIT(engine); 2877 } else if (INTEL_GEN(dev_priv) >= 6) {
2857 }
2858 } else if (INTEL_INFO(dev)->gen >= 6) {
2859 engine->init_context = intel_rcs_ctx_init; 2878 engine->init_context = intel_rcs_ctx_init;
2860 engine->add_request = gen6_add_request;
2861 engine->flush = gen7_render_ring_flush; 2879 engine->flush = gen7_render_ring_flush;
2862 if (INTEL_INFO(dev)->gen == 6) 2880 if (IS_GEN6(dev_priv))
2863 engine->flush = gen6_render_ring_flush; 2881 engine->flush = gen6_render_ring_flush;
2864 engine->irq_get = gen6_ring_get_irq; 2882 } else if (IS_GEN5(dev_priv)) {
2865 engine->irq_put = gen6_ring_put_irq;
2866 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2867 engine->irq_seqno_barrier = gen6_seqno_barrier;
2868 engine->get_seqno = ring_get_seqno;
2869 engine->set_seqno = ring_set_seqno;
2870 if (i915_semaphore_is_enabled(dev)) {
2871 engine->semaphore.sync_to = gen6_ring_sync;
2872 engine->semaphore.signal = gen6_signal;
2873 /*
2874 * The current semaphore is only applied on pre-gen8
2875 * platform. And there is no VCS2 ring on the pre-gen8
2876 * platform. So the semaphore between RCS and VCS2 is
2877 * initialized as INVALID. Gen8 will initialize the
2878 * sema between VCS2 and RCS later.
2879 */
2880 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2881 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
2882 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
2883 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
2884 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2885 engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2886 engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
2887 engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
2888 engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
2889 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2890 }
2891 } else if (IS_GEN5(dev)) {
2892 engine->add_request = pc_render_add_request;
2893 engine->flush = gen4_render_ring_flush; 2883 engine->flush = gen4_render_ring_flush;
2894 engine->get_seqno = pc_render_get_seqno;
2895 engine->set_seqno = pc_render_set_seqno;
2896 engine->irq_get = gen5_ring_get_irq;
2897 engine->irq_put = gen5_ring_put_irq;
2898 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
2899 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
2900 } else { 2884 } else {
2901 engine->add_request = i9xx_add_request; 2885 if (INTEL_GEN(dev_priv) < 4)
2902 if (INTEL_INFO(dev)->gen < 4)
2903 engine->flush = gen2_render_ring_flush; 2886 engine->flush = gen2_render_ring_flush;
2904 else 2887 else
2905 engine->flush = gen4_render_ring_flush; 2888 engine->flush = gen4_render_ring_flush;
2906 engine->get_seqno = ring_get_seqno;
2907 engine->set_seqno = ring_set_seqno;
2908 if (IS_GEN2(dev)) {
2909 engine->irq_get = i8xx_ring_get_irq;
2910 engine->irq_put = i8xx_ring_put_irq;
2911 } else {
2912 engine->irq_get = i9xx_ring_get_irq;
2913 engine->irq_put = i9xx_ring_put_irq;
2914 }
2915 engine->irq_enable_mask = I915_USER_INTERRUPT; 2889 engine->irq_enable_mask = I915_USER_INTERRUPT;
2916 } 2890 }
2917 engine->write_tail = ring_write_tail;
2918 2891
2919 if (IS_HASWELL(dev)) 2892 if (IS_HASWELL(dev_priv))
2920 engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; 2893 engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
2921 else if (IS_GEN8(dev)) 2894
2922 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2923 else if (INTEL_INFO(dev)->gen >= 6)
2924 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2925 else if (INTEL_INFO(dev)->gen >= 4)
2926 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
2927 else if (IS_I830(dev) || IS_845G(dev))
2928 engine->dispatch_execbuffer = i830_dispatch_execbuffer;
2929 else
2930 engine->dispatch_execbuffer = i915_dispatch_execbuffer;
2931 engine->init_hw = init_render_ring; 2895 engine->init_hw = init_render_ring;
2932 engine->cleanup = render_ring_cleanup; 2896 engine->cleanup = render_ring_cleanup;
2933 2897
2934 /* Workaround batchbuffer to combat CS tlb bug. */
2935 if (HAS_BROKEN_CS_TLB(dev)) {
2936 obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
2937 if (obj == NULL) {
2938 DRM_ERROR("Failed to allocate batch bo\n");
2939 return -ENOMEM;
2940 }
2941
2942 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
2943 if (ret != 0) {
2944 drm_gem_object_unreference(&obj->base);
2945 DRM_ERROR("Failed to ping batch bo\n");
2946 return ret;
2947 }
2948
2949 engine->scratch.obj = obj;
2950 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2951 }
2952
2953 ret = intel_init_ring_buffer(dev, engine); 2898 ret = intel_init_ring_buffer(dev, engine);
2954 if (ret) 2899 if (ret)
2955 return ret; 2900 return ret;
2956 2901
2957 if (INTEL_INFO(dev)->gen >= 5) { 2902 if (INTEL_GEN(dev_priv) >= 6) {
2958 ret = intel_init_pipe_control(engine); 2903 ret = intel_init_pipe_control(engine, 4096);
2904 if (ret)
2905 return ret;
2906 } else if (HAS_BROKEN_CS_TLB(dev_priv)) {
2907 ret = intel_init_pipe_control(engine, I830_WA_SIZE);
2959 if (ret) 2908 if (ret)
2960 return ret; 2909 return ret;
2961 } 2910 }
@@ -2965,7 +2914,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
2965 2914
2966int intel_init_bsd_ring_buffer(struct drm_device *dev) 2915int intel_init_bsd_ring_buffer(struct drm_device *dev)
2967{ 2916{
2968 struct drm_i915_private *dev_priv = dev->dev_private; 2917 struct drm_i915_private *dev_priv = to_i915(dev);
2969 struct intel_engine_cs *engine = &dev_priv->engine[VCS]; 2918 struct intel_engine_cs *engine = &dev_priv->engine[VCS];
2970 2919
2971 engine->name = "bsd ring"; 2920 engine->name = "bsd ring";
@@ -2973,68 +2922,27 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
2973 engine->exec_id = I915_EXEC_BSD; 2922 engine->exec_id = I915_EXEC_BSD;
2974 engine->hw_id = 1; 2923 engine->hw_id = 1;
2975 2924
2976 engine->write_tail = ring_write_tail; 2925 intel_ring_default_vfuncs(dev_priv, engine);
2977 if (INTEL_INFO(dev)->gen >= 6) { 2926
2927 if (INTEL_GEN(dev_priv) >= 6) {
2978 engine->mmio_base = GEN6_BSD_RING_BASE; 2928 engine->mmio_base = GEN6_BSD_RING_BASE;
2979 /* gen6 bsd needs a special wa for tail updates */ 2929 /* gen6 bsd needs a special wa for tail updates */
2980 if (IS_GEN6(dev)) 2930 if (IS_GEN6(dev_priv))
2981 engine->write_tail = gen6_bsd_ring_write_tail; 2931 engine->write_tail = gen6_bsd_ring_write_tail;
2982 engine->flush = gen6_bsd_ring_flush; 2932 engine->flush = gen6_bsd_ring_flush;
2983 engine->add_request = gen6_add_request; 2933 if (INTEL_GEN(dev_priv) >= 8)
2984 engine->irq_seqno_barrier = gen6_seqno_barrier;
2985 engine->get_seqno = ring_get_seqno;
2986 engine->set_seqno = ring_set_seqno;
2987 if (INTEL_INFO(dev)->gen >= 8) {
2988 engine->irq_enable_mask = 2934 engine->irq_enable_mask =
2989 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; 2935 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2990 engine->irq_get = gen8_ring_get_irq; 2936 else
2991 engine->irq_put = gen8_ring_put_irq;
2992 engine->dispatch_execbuffer =
2993 gen8_ring_dispatch_execbuffer;
2994 if (i915_semaphore_is_enabled(dev)) {
2995 engine->semaphore.sync_to = gen8_ring_sync;
2996 engine->semaphore.signal = gen8_xcs_signal;
2997 GEN8_RING_SEMAPHORE_INIT(engine);
2998 }
2999 } else {
3000 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 2937 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
3001 engine->irq_get = gen6_ring_get_irq;
3002 engine->irq_put = gen6_ring_put_irq;
3003 engine->dispatch_execbuffer =
3004 gen6_ring_dispatch_execbuffer;
3005 if (i915_semaphore_is_enabled(dev)) {
3006 engine->semaphore.sync_to = gen6_ring_sync;
3007 engine->semaphore.signal = gen6_signal;
3008 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
3009 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
3010 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
3011 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
3012 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3013 engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
3014 engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
3015 engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
3016 engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
3017 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3018 }
3019 }
3020 } else { 2938 } else {
3021 engine->mmio_base = BSD_RING_BASE; 2939 engine->mmio_base = BSD_RING_BASE;
3022 engine->flush = bsd_ring_flush; 2940 engine->flush = bsd_ring_flush;
3023 engine->add_request = i9xx_add_request; 2941 if (IS_GEN5(dev_priv))
3024 engine->get_seqno = ring_get_seqno;
3025 engine->set_seqno = ring_set_seqno;
3026 if (IS_GEN5(dev)) {
3027 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 2942 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3028 engine->irq_get = gen5_ring_get_irq; 2943 else
3029 engine->irq_put = gen5_ring_put_irq;
3030 } else {
3031 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 2944 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
3032 engine->irq_get = i9xx_ring_get_irq;
3033 engine->irq_put = i9xx_ring_put_irq;
3034 }
3035 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
3036 } 2945 }
3037 engine->init_hw = init_ring_common;
3038 2946
3039 return intel_init_ring_buffer(dev, engine); 2947 return intel_init_ring_buffer(dev, engine);
3040} 2948}
@@ -3044,147 +2952,70 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
3044 */ 2952 */
3045int intel_init_bsd2_ring_buffer(struct drm_device *dev) 2953int intel_init_bsd2_ring_buffer(struct drm_device *dev)
3046{ 2954{
3047 struct drm_i915_private *dev_priv = dev->dev_private; 2955 struct drm_i915_private *dev_priv = to_i915(dev);
3048 struct intel_engine_cs *engine = &dev_priv->engine[VCS2]; 2956 struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
3049 2957
3050 engine->name = "bsd2 ring"; 2958 engine->name = "bsd2 ring";
3051 engine->id = VCS2; 2959 engine->id = VCS2;
3052 engine->exec_id = I915_EXEC_BSD; 2960 engine->exec_id = I915_EXEC_BSD;
3053 engine->hw_id = 4; 2961 engine->hw_id = 4;
3054
3055 engine->write_tail = ring_write_tail;
3056 engine->mmio_base = GEN8_BSD2_RING_BASE; 2962 engine->mmio_base = GEN8_BSD2_RING_BASE;
2963
2964 intel_ring_default_vfuncs(dev_priv, engine);
2965
3057 engine->flush = gen6_bsd_ring_flush; 2966 engine->flush = gen6_bsd_ring_flush;
3058 engine->add_request = gen6_add_request;
3059 engine->irq_seqno_barrier = gen6_seqno_barrier;
3060 engine->get_seqno = ring_get_seqno;
3061 engine->set_seqno = ring_set_seqno;
3062 engine->irq_enable_mask = 2967 engine->irq_enable_mask =
3063 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT; 2968 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
3064 engine->irq_get = gen8_ring_get_irq;
3065 engine->irq_put = gen8_ring_put_irq;
3066 engine->dispatch_execbuffer =
3067 gen8_ring_dispatch_execbuffer;
3068 if (i915_semaphore_is_enabled(dev)) {
3069 engine->semaphore.sync_to = gen8_ring_sync;
3070 engine->semaphore.signal = gen8_xcs_signal;
3071 GEN8_RING_SEMAPHORE_INIT(engine);
3072 }
3073 engine->init_hw = init_ring_common;
3074 2969
3075 return intel_init_ring_buffer(dev, engine); 2970 return intel_init_ring_buffer(dev, engine);
3076} 2971}
3077 2972
3078int intel_init_blt_ring_buffer(struct drm_device *dev) 2973int intel_init_blt_ring_buffer(struct drm_device *dev)
3079{ 2974{
3080 struct drm_i915_private *dev_priv = dev->dev_private; 2975 struct drm_i915_private *dev_priv = to_i915(dev);
3081 struct intel_engine_cs *engine = &dev_priv->engine[BCS]; 2976 struct intel_engine_cs *engine = &dev_priv->engine[BCS];
3082 2977
3083 engine->name = "blitter ring"; 2978 engine->name = "blitter ring";
3084 engine->id = BCS; 2979 engine->id = BCS;
3085 engine->exec_id = I915_EXEC_BLT; 2980 engine->exec_id = I915_EXEC_BLT;
3086 engine->hw_id = 2; 2981 engine->hw_id = 2;
3087
3088 engine->mmio_base = BLT_RING_BASE; 2982 engine->mmio_base = BLT_RING_BASE;
3089 engine->write_tail = ring_write_tail; 2983
2984 intel_ring_default_vfuncs(dev_priv, engine);
2985
3090 engine->flush = gen6_ring_flush; 2986 engine->flush = gen6_ring_flush;
3091 engine->add_request = gen6_add_request; 2987 if (INTEL_GEN(dev_priv) >= 8)
3092 engine->irq_seqno_barrier = gen6_seqno_barrier;
3093 engine->get_seqno = ring_get_seqno;
3094 engine->set_seqno = ring_set_seqno;
3095 if (INTEL_INFO(dev)->gen >= 8) {
3096 engine->irq_enable_mask = 2988 engine->irq_enable_mask =
3097 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; 2989 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
3098 engine->irq_get = gen8_ring_get_irq; 2990 else
3099 engine->irq_put = gen8_ring_put_irq;
3100 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3101 if (i915_semaphore_is_enabled(dev)) {
3102 engine->semaphore.sync_to = gen8_ring_sync;
3103 engine->semaphore.signal = gen8_xcs_signal;
3104 GEN8_RING_SEMAPHORE_INIT(engine);
3105 }
3106 } else {
3107 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 2991 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
3108 engine->irq_get = gen6_ring_get_irq;
3109 engine->irq_put = gen6_ring_put_irq;
3110 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3111 if (i915_semaphore_is_enabled(dev)) {
3112 engine->semaphore.signal = gen6_signal;
3113 engine->semaphore.sync_to = gen6_ring_sync;
3114 /*
3115 * The current semaphore is only applied on pre-gen8
3116 * platform. And there is no VCS2 ring on the pre-gen8
3117 * platform. So the semaphore between BCS and VCS2 is
3118 * initialized as INVALID. Gen8 will initialize the
3119 * sema between BCS and VCS2 later.
3120 */
3121 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
3122 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
3123 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
3124 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
3125 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3126 engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
3127 engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
3128 engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
3129 engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
3130 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3131 }
3132 }
3133 engine->init_hw = init_ring_common;
3134 2992
3135 return intel_init_ring_buffer(dev, engine); 2993 return intel_init_ring_buffer(dev, engine);
3136} 2994}
3137 2995
3138int intel_init_vebox_ring_buffer(struct drm_device *dev) 2996int intel_init_vebox_ring_buffer(struct drm_device *dev)
3139{ 2997{
3140 struct drm_i915_private *dev_priv = dev->dev_private; 2998 struct drm_i915_private *dev_priv = to_i915(dev);
3141 struct intel_engine_cs *engine = &dev_priv->engine[VECS]; 2999 struct intel_engine_cs *engine = &dev_priv->engine[VECS];
3142 3000
3143 engine->name = "video enhancement ring"; 3001 engine->name = "video enhancement ring";
3144 engine->id = VECS; 3002 engine->id = VECS;
3145 engine->exec_id = I915_EXEC_VEBOX; 3003 engine->exec_id = I915_EXEC_VEBOX;
3146 engine->hw_id = 3; 3004 engine->hw_id = 3;
3147
3148 engine->mmio_base = VEBOX_RING_BASE; 3005 engine->mmio_base = VEBOX_RING_BASE;
3149 engine->write_tail = ring_write_tail; 3006
3007 intel_ring_default_vfuncs(dev_priv, engine);
3008
3150 engine->flush = gen6_ring_flush; 3009 engine->flush = gen6_ring_flush;
3151 engine->add_request = gen6_add_request;
3152 engine->irq_seqno_barrier = gen6_seqno_barrier;
3153 engine->get_seqno = ring_get_seqno;
3154 engine->set_seqno = ring_set_seqno;
3155 3010
3156 if (INTEL_INFO(dev)->gen >= 8) { 3011 if (INTEL_GEN(dev_priv) >= 8) {
3157 engine->irq_enable_mask = 3012 engine->irq_enable_mask =
3158 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT; 3013 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
3159 engine->irq_get = gen8_ring_get_irq;
3160 engine->irq_put = gen8_ring_put_irq;
3161 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3162 if (i915_semaphore_is_enabled(dev)) {
3163 engine->semaphore.sync_to = gen8_ring_sync;
3164 engine->semaphore.signal = gen8_xcs_signal;
3165 GEN8_RING_SEMAPHORE_INIT(engine);
3166 }
3167 } else { 3014 } else {
3168 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 3015 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
3169 engine->irq_get = hsw_vebox_get_irq; 3016 engine->irq_enable = hsw_vebox_irq_enable;
3170 engine->irq_put = hsw_vebox_put_irq; 3017 engine->irq_disable = hsw_vebox_irq_disable;
3171 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3172 if (i915_semaphore_is_enabled(dev)) {
3173 engine->semaphore.sync_to = gen6_ring_sync;
3174 engine->semaphore.signal = gen6_signal;
3175 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
3176 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
3177 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
3178 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
3179 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3180 engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
3181 engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
3182 engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
3183 engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
3184 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3185 }
3186 } 3018 }
3187 engine->init_hw = init_ring_common;
3188 3019
3189 return intel_init_ring_buffer(dev, engine); 3020 return intel_init_ring_buffer(dev, engine);
3190} 3021}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-22 03:39:24 -0500