aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/intel_workarounds.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/intel_workarounds.c')
-rw-r--r--drivers/gpu/drm/i915/intel_workarounds.c591
1 files changed, 390 insertions, 201 deletions
diff --git a/drivers/gpu/drm/i915/intel_workarounds.c b/drivers/gpu/drm/i915/intel_workarounds.c
index 4bcdeaf8d98f..6e580891db96 100644
--- a/drivers/gpu/drm/i915/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/intel_workarounds.c
@@ -48,6 +48,20 @@
48 * - Public functions to init or apply the given workaround type. 48 * - Public functions to init or apply the given workaround type.
49 */ 49 */
50 50
51static void wa_init_start(struct i915_wa_list *wal, const char *name)
52{
53 wal->name = name;
54}
55
56static void wa_init_finish(struct i915_wa_list *wal)
57{
58 if (!wal->count)
59 return;
60
61 DRM_DEBUG_DRIVER("Initialized %u %s workarounds\n",
62 wal->count, wal->name);
63}
64
51static void wa_add(struct drm_i915_private *i915, 65static void wa_add(struct drm_i915_private *i915,
52 i915_reg_t reg, const u32 mask, const u32 val) 66 i915_reg_t reg, const u32 mask, const u32 val)
53{ 67{
@@ -580,160 +594,175 @@ int intel_ctx_workarounds_emit(struct i915_request *rq)
580 return 0; 594 return 0;
581} 595}
582 596
583static void bdw_gt_workarounds_apply(struct drm_i915_private *dev_priv) 597static void
598wal_add(struct i915_wa_list *wal, const struct i915_wa *wa)
599{
600 const unsigned int grow = 1 << 4;
601
602 GEM_BUG_ON(!is_power_of_2(grow));
603
604 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
605 struct i915_wa *list;
606
607 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
608 GFP_KERNEL);
609 if (!list) {
610 DRM_ERROR("No space for workaround init!\n");
611 return;
612 }
613
614 if (wal->list)
615 memcpy(list, wal->list, sizeof(*wa) * wal->count);
616
617 wal->list = list;
618 }
619
620 wal->list[wal->count++] = *wa;
621}
622
623static void
624wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
625{
626 struct i915_wa wa = {
627 .reg = reg,
628 .mask = val,
629 .val = _MASKED_BIT_ENABLE(val)
630 };
631
632 wal_add(wal, &wa);
633}
634
635static void
636wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask,
637 u32 val)
584{ 638{
639 struct i915_wa wa = {
640 .reg = reg,
641 .mask = mask,
642 .val = val
643 };
644
645 wal_add(wal, &wa);
585} 646}
586 647
587static void chv_gt_workarounds_apply(struct drm_i915_private *dev_priv) 648static void
649wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
588{ 650{
651 wa_write_masked_or(wal, reg, ~0, val);
589} 652}
590 653
591static void gen9_gt_workarounds_apply(struct drm_i915_private *dev_priv) 654static void
655wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
592{ 656{
593 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */ 657 wa_write_masked_or(wal, reg, val, val);
594 I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, 658}
595 _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
596 659
597 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */ 660static void gen9_gt_workarounds_init(struct drm_i915_private *i915)
598 I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) | 661{
599 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); 662 struct i915_wa_list *wal = &i915->gt_wa_list;
600 663
601 /* WaDisableKillLogic:bxt,skl,kbl */ 664 /* WaDisableKillLogic:bxt,skl,kbl */
602 if (!IS_COFFEELAKE(dev_priv)) 665 if (!IS_COFFEELAKE(i915))
603 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | 666 wa_write_or(wal,
604 ECOCHK_DIS_TLB); 667 GAM_ECOCHK,
668 ECOCHK_DIS_TLB);
605 669
606 if (HAS_LLC(dev_priv)) { 670 if (HAS_LLC(i915)) {
607 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl 671 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
608 * 672 *
609 * Must match Display Engine. See 673 * Must match Display Engine. See
610 * WaCompressedResourceDisplayNewHashMode. 674 * WaCompressedResourceDisplayNewHashMode.
611 */ 675 */
612 I915_WRITE(MMCD_MISC_CTRL, 676 wa_write_or(wal,
613 I915_READ(MMCD_MISC_CTRL) | 677 MMCD_MISC_CTRL,
614 MMCD_PCLA | 678 MMCD_PCLA | MMCD_HOTSPOT_EN);
615 MMCD_HOTSPOT_EN);
616 } 679 }
617 680
618 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */ 681 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
619 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | 682 wa_write_or(wal,
620 BDW_DISABLE_HDC_INVALIDATION); 683 GAM_ECOCHK,
621 684 BDW_DISABLE_HDC_INVALIDATION);
622 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
623 if (IS_GEN9_LP(dev_priv)) {
624 u32 val = I915_READ(GEN8_L3SQCREG1);
625
626 val &= ~L3_PRIO_CREDITS_MASK;
627 val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
628 I915_WRITE(GEN8_L3SQCREG1, val);
629 }
630
631 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
632 I915_WRITE(GEN8_L3SQCREG4,
633 I915_READ(GEN8_L3SQCREG4) | GEN8_LQSC_FLUSH_COHERENT_LINES);
634
635 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
636 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
637 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
638} 685}
639 686
640static void skl_gt_workarounds_apply(struct drm_i915_private *dev_priv) 687static void skl_gt_workarounds_init(struct drm_i915_private *i915)
641{ 688{
642 gen9_gt_workarounds_apply(dev_priv); 689 struct i915_wa_list *wal = &i915->gt_wa_list;
643 690
644 /* WaEnableGapsTsvCreditFix:skl */ 691 gen9_gt_workarounds_init(i915);
645 I915_WRITE(GEN8_GARBCNTL,
646 I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
647 692
648 /* WaDisableGafsUnitClkGating:skl */ 693 /* WaDisableGafsUnitClkGating:skl */
649 I915_WRITE(GEN7_UCGCTL4, 694 wa_write_or(wal,
650 I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 695 GEN7_UCGCTL4,
696 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
651 697
652 /* WaInPlaceDecompressionHang:skl */ 698 /* WaInPlaceDecompressionHang:skl */
653 if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER)) 699 if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
654 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, 700 wa_write_or(wal,
655 I915_READ(GEN9_GAMT_ECO_REG_RW_IA) | 701 GEN9_GAMT_ECO_REG_RW_IA,
656 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 702 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
657} 703}
658 704
659static void bxt_gt_workarounds_apply(struct drm_i915_private *dev_priv) 705static void bxt_gt_workarounds_init(struct drm_i915_private *i915)
660{ 706{
661 gen9_gt_workarounds_apply(dev_priv); 707 struct i915_wa_list *wal = &i915->gt_wa_list;
662 708
663 /* WaDisablePooledEuLoadBalancingFix:bxt */ 709 gen9_gt_workarounds_init(i915);
664 I915_WRITE(FF_SLICE_CS_CHICKEN2,
665 _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
666 710
667 /* WaInPlaceDecompressionHang:bxt */ 711 /* WaInPlaceDecompressionHang:bxt */
668 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, 712 wa_write_or(wal,
669 I915_READ(GEN9_GAMT_ECO_REG_RW_IA) | 713 GEN9_GAMT_ECO_REG_RW_IA,
670 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 714 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
671} 715}
672 716
673static void kbl_gt_workarounds_apply(struct drm_i915_private *dev_priv) 717static void kbl_gt_workarounds_init(struct drm_i915_private *i915)
674{ 718{
675 gen9_gt_workarounds_apply(dev_priv); 719 struct i915_wa_list *wal = &i915->gt_wa_list;
676 720
677 /* WaEnableGapsTsvCreditFix:kbl */ 721 gen9_gt_workarounds_init(i915);
678 I915_WRITE(GEN8_GARBCNTL,
679 I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
680 722
681 /* WaDisableDynamicCreditSharing:kbl */ 723 /* WaDisableDynamicCreditSharing:kbl */
682 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) 724 if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
683 I915_WRITE(GAMT_CHKN_BIT_REG, 725 wa_write_or(wal,
684 I915_READ(GAMT_CHKN_BIT_REG) | 726 GAMT_CHKN_BIT_REG,
685 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING); 727 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
686 728
687 /* WaDisableGafsUnitClkGating:kbl */ 729 /* WaDisableGafsUnitClkGating:kbl */
688 I915_WRITE(GEN7_UCGCTL4, 730 wa_write_or(wal,
689 I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 731 GEN7_UCGCTL4,
732 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
690 733
691 /* WaInPlaceDecompressionHang:kbl */ 734 /* WaInPlaceDecompressionHang:kbl */
692 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, 735 wa_write_or(wal,
693 I915_READ(GEN9_GAMT_ECO_REG_RW_IA) | 736 GEN9_GAMT_ECO_REG_RW_IA,
694 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 737 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
695
696 /* WaKBLVECSSemaphoreWaitPoll:kbl */
697 if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_E0)) {
698 struct intel_engine_cs *engine;
699 unsigned int tmp;
700
701 for_each_engine(engine, dev_priv, tmp) {
702 if (engine->id == RCS)
703 continue;
704
705 I915_WRITE(RING_SEMA_WAIT_POLL(engine->mmio_base), 1);
706 }
707 }
708} 738}
709 739
710static void glk_gt_workarounds_apply(struct drm_i915_private *dev_priv) 740static void glk_gt_workarounds_init(struct drm_i915_private *i915)
711{ 741{
712 gen9_gt_workarounds_apply(dev_priv); 742 gen9_gt_workarounds_init(i915);
713} 743}
714 744
715static void cfl_gt_workarounds_apply(struct drm_i915_private *dev_priv) 745static void cfl_gt_workarounds_init(struct drm_i915_private *i915)
716{ 746{
717 gen9_gt_workarounds_apply(dev_priv); 747 struct i915_wa_list *wal = &i915->gt_wa_list;
718 748
719 /* WaEnableGapsTsvCreditFix:cfl */ 749 gen9_gt_workarounds_init(i915);
720 I915_WRITE(GEN8_GARBCNTL,
721 I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
722 750
723 /* WaDisableGafsUnitClkGating:cfl */ 751 /* WaDisableGafsUnitClkGating:cfl */
724 I915_WRITE(GEN7_UCGCTL4, 752 wa_write_or(wal,
725 I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 753 GEN7_UCGCTL4,
754 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
726 755
727 /* WaInPlaceDecompressionHang:cfl */ 756 /* WaInPlaceDecompressionHang:cfl */
728 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, 757 wa_write_or(wal,
729 I915_READ(GEN9_GAMT_ECO_REG_RW_IA) | 758 GEN9_GAMT_ECO_REG_RW_IA,
730 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 759 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
731} 760}
732 761
733static void wa_init_mcr(struct drm_i915_private *dev_priv) 762static void wa_init_mcr(struct drm_i915_private *dev_priv)
734{ 763{
735 const struct sseu_dev_info *sseu = &(INTEL_INFO(dev_priv)->sseu); 764 const struct sseu_dev_info *sseu = &(INTEL_INFO(dev_priv)->sseu);
736 u32 mcr; 765 struct i915_wa_list *wal = &dev_priv->gt_wa_list;
737 u32 mcr_slice_subslice_mask; 766 u32 mcr_slice_subslice_mask;
738 767
739 /* 768 /*
@@ -770,8 +799,6 @@ static void wa_init_mcr(struct drm_i915_private *dev_priv)
770 WARN_ON((enabled_mask & disabled_mask) != enabled_mask); 799 WARN_ON((enabled_mask & disabled_mask) != enabled_mask);
771 } 800 }
772 801
773 mcr = I915_READ(GEN8_MCR_SELECTOR);
774
775 if (INTEL_GEN(dev_priv) >= 11) 802 if (INTEL_GEN(dev_priv) >= 11)
776 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK | 803 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
777 GEN11_MCR_SUBSLICE_MASK; 804 GEN11_MCR_SUBSLICE_MASK;
@@ -789,148 +816,170 @@ static void wa_init_mcr(struct drm_i915_private *dev_priv)
789 * occasions, such as INSTDONE, where this value is dependent 816 * occasions, such as INSTDONE, where this value is dependent
790 * on s/ss combo, the read should be done with read_subslice_reg. 817 * on s/ss combo, the read should be done with read_subslice_reg.
791 */ 818 */
792 mcr &= ~mcr_slice_subslice_mask; 819 wa_write_masked_or(wal,
793 mcr |= intel_calculate_mcr_s_ss_select(dev_priv); 820 GEN8_MCR_SELECTOR,
794 I915_WRITE(GEN8_MCR_SELECTOR, mcr); 821 mcr_slice_subslice_mask,
822 intel_calculate_mcr_s_ss_select(dev_priv));
795} 823}
796 824
797static void cnl_gt_workarounds_apply(struct drm_i915_private *dev_priv) 825static void cnl_gt_workarounds_init(struct drm_i915_private *i915)
798{ 826{
799 wa_init_mcr(dev_priv); 827 struct i915_wa_list *wal = &i915->gt_wa_list;
828
829 wa_init_mcr(i915);
800 830
801 /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */ 831 /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
802 if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0)) 832 if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
803 I915_WRITE(GAMT_CHKN_BIT_REG, 833 wa_write_or(wal,
804 I915_READ(GAMT_CHKN_BIT_REG) | 834 GAMT_CHKN_BIT_REG,
805 GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT); 835 GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
806 836
807 /* WaInPlaceDecompressionHang:cnl */ 837 /* WaInPlaceDecompressionHang:cnl */
808 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, 838 wa_write_or(wal,
809 I915_READ(GEN9_GAMT_ECO_REG_RW_IA) | 839 GEN9_GAMT_ECO_REG_RW_IA,
810 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 840 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
811
812 /* WaEnablePreemptionGranularityControlByUMD:cnl */
813 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
814 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
815} 841}
816 842
817static void icl_gt_workarounds_apply(struct drm_i915_private *dev_priv) 843static void icl_gt_workarounds_init(struct drm_i915_private *i915)
818{ 844{
819 wa_init_mcr(dev_priv); 845 struct i915_wa_list *wal = &i915->gt_wa_list;
820 846
821 /* This is not an Wa. Enable for better image quality */ 847 wa_init_mcr(i915);
822 I915_WRITE(_3D_CHICKEN3,
823 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
824 848
825 /* WaInPlaceDecompressionHang:icl */ 849 /* WaInPlaceDecompressionHang:icl */
826 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, I915_READ(GEN9_GAMT_ECO_REG_RW_IA) | 850 wa_write_or(wal,
827 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 851 GEN9_GAMT_ECO_REG_RW_IA,
828 852 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
829 /* WaPipelineFlushCoherentLines:icl */
830 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
831 GEN8_LQSC_FLUSH_COHERENT_LINES);
832
833 /* Wa_1405543622:icl
834 * Formerly known as WaGAPZPriorityScheme
835 */
836 I915_WRITE(GEN8_GARBCNTL, I915_READ(GEN8_GARBCNTL) |
837 GEN11_ARBITRATION_PRIO_ORDER_MASK);
838
839 /* Wa_1604223664:icl
840 * Formerly known as WaL3BankAddressHashing
841 */
842 I915_WRITE(GEN8_GARBCNTL,
843 (I915_READ(GEN8_GARBCNTL) & ~GEN11_HASH_CTRL_EXCL_MASK) |
844 GEN11_HASH_CTRL_EXCL_BIT0);
845 I915_WRITE(GEN11_GLBLINVL,
846 (I915_READ(GEN11_GLBLINVL) & ~GEN11_BANK_HASH_ADDR_EXCL_MASK) |
847 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
848 853
849 /* WaModifyGamTlbPartitioning:icl */ 854 /* WaModifyGamTlbPartitioning:icl */
850 I915_WRITE(GEN11_GACB_PERF_CTRL, 855 wa_write_masked_or(wal,
851 (I915_READ(GEN11_GACB_PERF_CTRL) & ~GEN11_HASH_CTRL_MASK) | 856 GEN11_GACB_PERF_CTRL,
852 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4); 857 GEN11_HASH_CTRL_MASK,
853 858 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
854 /* Wa_1405733216:icl
855 * Formerly known as WaDisableCleanEvicts
856 */
857 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
858 GEN11_LQSC_CLEAN_EVICT_DISABLE);
859 859
860 /* Wa_1405766107:icl 860 /* Wa_1405766107:icl
861 * Formerly known as WaCL2SFHalfMaxAlloc 861 * Formerly known as WaCL2SFHalfMaxAlloc
862 */ 862 */
863 I915_WRITE(GEN11_LSN_UNSLCVC, I915_READ(GEN11_LSN_UNSLCVC) | 863 wa_write_or(wal,
864 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC | 864 GEN11_LSN_UNSLCVC,
865 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC); 865 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
866 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
866 867
867 /* Wa_220166154:icl 868 /* Wa_220166154:icl
868 * Formerly known as WaDisCtxReload 869 * Formerly known as WaDisCtxReload
869 */ 870 */
870 I915_WRITE(GAMW_ECO_DEV_RW_IA_REG, I915_READ(GAMW_ECO_DEV_RW_IA_REG) | 871 wa_write_or(wal,
871 GAMW_ECO_DEV_CTX_RELOAD_DISABLE); 872 GEN8_GAMW_ECO_DEV_RW_IA,
873 GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
872 874
873 /* Wa_1405779004:icl (pre-prod) */ 875 /* Wa_1405779004:icl (pre-prod) */
874 if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_A0)) 876 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
875 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, 877 wa_write_or(wal,
876 I915_READ(SLICE_UNIT_LEVEL_CLKGATE) | 878 SLICE_UNIT_LEVEL_CLKGATE,
877 MSCUNIT_CLKGATE_DIS); 879 MSCUNIT_CLKGATE_DIS);
878 880
879 /* Wa_1406680159:icl */ 881 /* Wa_1406680159:icl */
880 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, 882 wa_write_or(wal,
881 I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE) | 883 SUBSLICE_UNIT_LEVEL_CLKGATE,
882 GWUNIT_CLKGATE_DIS); 884 GWUNIT_CLKGATE_DIS);
883
884 /* Wa_1604302699:icl */
885 I915_WRITE(GEN10_L3_CHICKEN_MODE_REGISTER,
886 I915_READ(GEN10_L3_CHICKEN_MODE_REGISTER) |
887 GEN11_I2M_WRITE_DISABLE);
888 885
889 /* Wa_1406838659:icl (pre-prod) */ 886 /* Wa_1406838659:icl (pre-prod) */
890 if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_B0)) 887 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
891 I915_WRITE(INF_UNIT_LEVEL_CLKGATE, 888 wa_write_or(wal,
892 I915_READ(INF_UNIT_LEVEL_CLKGATE) | 889 INF_UNIT_LEVEL_CLKGATE,
893 CGPSF_CLKGATE_DIS); 890 CGPSF_CLKGATE_DIS);
894
895 /* WaForwardProgressSoftReset:icl */
896 I915_WRITE(GEN10_SCRATCH_LNCF2,
897 I915_READ(GEN10_SCRATCH_LNCF2) |
898 PMFLUSHDONE_LNICRSDROP |
899 PMFLUSH_GAPL3UNBLOCK |
900 PMFLUSHDONE_LNEBLK);
901 891
902 /* Wa_1406463099:icl 892 /* Wa_1406463099:icl
903 * Formerly known as WaGamTlbPendError 893 * Formerly known as WaGamTlbPendError
904 */ 894 */
905 I915_WRITE(GAMT_CHKN_BIT_REG, 895 wa_write_or(wal,
906 I915_READ(GAMT_CHKN_BIT_REG) | 896 GAMT_CHKN_BIT_REG,
907 GAMT_CHKN_DISABLE_L3_COH_PIPE); 897 GAMT_CHKN_DISABLE_L3_COH_PIPE);
908} 898}
909 899
910void intel_gt_workarounds_apply(struct drm_i915_private *dev_priv) 900void intel_gt_init_workarounds(struct drm_i915_private *i915)
911{ 901{
912 if (INTEL_GEN(dev_priv) < 8) 902 struct i915_wa_list *wal = &i915->gt_wa_list;
903
904 wa_init_start(wal, "GT");
905
906 if (INTEL_GEN(i915) < 8)
913 return; 907 return;
914 else if (IS_BROADWELL(dev_priv)) 908 else if (IS_BROADWELL(i915))
915 bdw_gt_workarounds_apply(dev_priv); 909 return;
916 else if (IS_CHERRYVIEW(dev_priv)) 910 else if (IS_CHERRYVIEW(i915))
917 chv_gt_workarounds_apply(dev_priv); 911 return;
918 else if (IS_SKYLAKE(dev_priv)) 912 else if (IS_SKYLAKE(i915))
919 skl_gt_workarounds_apply(dev_priv); 913 skl_gt_workarounds_init(i915);
920 else if (IS_BROXTON(dev_priv)) 914 else if (IS_BROXTON(i915))
921 bxt_gt_workarounds_apply(dev_priv); 915 bxt_gt_workarounds_init(i915);
922 else if (IS_KABYLAKE(dev_priv)) 916 else if (IS_KABYLAKE(i915))
923 kbl_gt_workarounds_apply(dev_priv); 917 kbl_gt_workarounds_init(i915);
924 else if (IS_GEMINILAKE(dev_priv)) 918 else if (IS_GEMINILAKE(i915))
925 glk_gt_workarounds_apply(dev_priv); 919 glk_gt_workarounds_init(i915);
926 else if (IS_COFFEELAKE(dev_priv)) 920 else if (IS_COFFEELAKE(i915))
927 cfl_gt_workarounds_apply(dev_priv); 921 cfl_gt_workarounds_init(i915);
928 else if (IS_CANNONLAKE(dev_priv)) 922 else if (IS_CANNONLAKE(i915))
929 cnl_gt_workarounds_apply(dev_priv); 923 cnl_gt_workarounds_init(i915);
930 else if (IS_ICELAKE(dev_priv)) 924 else if (IS_ICELAKE(i915))
931 icl_gt_workarounds_apply(dev_priv); 925 icl_gt_workarounds_init(i915);
932 else 926 else
933 MISSING_CASE(INTEL_GEN(dev_priv)); 927 MISSING_CASE(INTEL_GEN(i915));
928
929 wa_init_finish(wal);
930}
931
932static enum forcewake_domains
933wal_get_fw_for_rmw(struct drm_i915_private *dev_priv,
934 const struct i915_wa_list *wal)
935{
936 enum forcewake_domains fw = 0;
937 struct i915_wa *wa;
938 unsigned int i;
939
940 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
941 fw |= intel_uncore_forcewake_for_reg(dev_priv,
942 wa->reg,
943 FW_REG_READ |
944 FW_REG_WRITE);
945
946 return fw;
947}
948
949static void
950wa_list_apply(struct drm_i915_private *dev_priv, const struct i915_wa_list *wal)
951{
952 enum forcewake_domains fw;
953 unsigned long flags;
954 struct i915_wa *wa;
955 unsigned int i;
956
957 if (!wal->count)
958 return;
959
960 fw = wal_get_fw_for_rmw(dev_priv, wal);
961
962 spin_lock_irqsave(&dev_priv->uncore.lock, flags);
963 intel_uncore_forcewake_get__locked(dev_priv, fw);
964
965 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
966 u32 val = I915_READ_FW(wa->reg);
967
968 val &= ~wa->mask;
969 val |= wa->val;
970
971 I915_WRITE_FW(wa->reg, val);
972 }
973
974 intel_uncore_forcewake_put__locked(dev_priv, fw);
975 spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
976
977 DRM_DEBUG_DRIVER("Applied %u %s workarounds\n", wal->count, wal->name);
978}
979
980void intel_gt_apply_workarounds(struct drm_i915_private *dev_priv)
981{
982 wa_list_apply(dev_priv, &dev_priv->gt_wa_list);
934} 983}
935 984
936struct whitelist { 985struct whitelist {
@@ -1077,6 +1126,146 @@ void intel_whitelist_workarounds_apply(struct intel_engine_cs *engine)
1077 whitelist_apply(engine, whitelist_build(engine, &w)); 1126 whitelist_apply(engine, whitelist_build(engine, &w));
1078} 1127}
1079 1128
1129static void rcs_engine_wa_init(struct intel_engine_cs *engine)
1130{
1131 struct drm_i915_private *i915 = engine->i915;
1132 struct i915_wa_list *wal = &engine->wa_list;
1133
1134 if (IS_ICELAKE(i915)) {
1135 /* This is not an Wa. Enable for better image quality */
1136 wa_masked_en(wal,
1137 _3D_CHICKEN3,
1138 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1139
1140 /* WaPipelineFlushCoherentLines:icl */
1141 wa_write_or(wal,
1142 GEN8_L3SQCREG4,
1143 GEN8_LQSC_FLUSH_COHERENT_LINES);
1144
1145 /*
1146 * Wa_1405543622:icl
1147 * Formerly known as WaGAPZPriorityScheme
1148 */
1149 wa_write_or(wal,
1150 GEN8_GARBCNTL,
1151 GEN11_ARBITRATION_PRIO_ORDER_MASK);
1152
1153 /*
1154 * Wa_1604223664:icl
1155 * Formerly known as WaL3BankAddressHashing
1156 */
1157 wa_write_masked_or(wal,
1158 GEN8_GARBCNTL,
1159 GEN11_HASH_CTRL_EXCL_MASK,
1160 GEN11_HASH_CTRL_EXCL_BIT0);
1161 wa_write_masked_or(wal,
1162 GEN11_GLBLINVL,
1163 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1164 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1165
1166 /*
1167 * Wa_1405733216:icl
1168 * Formerly known as WaDisableCleanEvicts
1169 */
1170 wa_write_or(wal,
1171 GEN8_L3SQCREG4,
1172 GEN11_LQSC_CLEAN_EVICT_DISABLE);
1173
1174 /* Wa_1604302699:icl */
1175 wa_write_or(wal,
1176 GEN10_L3_CHICKEN_MODE_REGISTER,
1177 GEN11_I2M_WRITE_DISABLE);
1178
1179 /* WaForwardProgressSoftReset:icl */
1180 wa_write_or(wal,
1181 GEN10_SCRATCH_LNCF2,
1182 PMFLUSHDONE_LNICRSDROP |
1183 PMFLUSH_GAPL3UNBLOCK |
1184 PMFLUSHDONE_LNEBLK);
1185 }
1186
1187 if (IS_GEN9(i915) || IS_CANNONLAKE(i915)) {
1188 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,cnl */
1189 wa_masked_en(wal,
1190 GEN7_FF_SLICE_CS_CHICKEN1,
1191 GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1192 }
1193
1194 if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1195 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1196 wa_write_or(wal,
1197 GEN8_GARBCNTL,
1198 GEN9_GAPS_TSV_CREDIT_DISABLE);
1199 }
1200
1201 if (IS_BROXTON(i915)) {
1202 /* WaDisablePooledEuLoadBalancingFix:bxt */
1203 wa_masked_en(wal,
1204 FF_SLICE_CS_CHICKEN2,
1205 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1206 }
1207
1208 if (IS_GEN9(i915)) {
1209 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1210 wa_masked_en(wal,
1211 GEN9_CSFE_CHICKEN1_RCS,
1212 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1213
1214 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1215 wa_write_or(wal,
1216 BDW_SCRATCH1,
1217 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1218
1219 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1220 if (IS_GEN9_LP(i915))
1221 wa_write_masked_or(wal,
1222 GEN8_L3SQCREG1,
1223 L3_PRIO_CREDITS_MASK,
1224 L3_GENERAL_PRIO_CREDITS(62) |
1225 L3_HIGH_PRIO_CREDITS(2));
1226
1227 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1228 wa_write_or(wal,
1229 GEN8_L3SQCREG4,
1230 GEN8_LQSC_FLUSH_COHERENT_LINES);
1231 }
1232}
1233
1234static void xcs_engine_wa_init(struct intel_engine_cs *engine)
1235{
1236 struct drm_i915_private *i915 = engine->i915;
1237 struct i915_wa_list *wal = &engine->wa_list;
1238
1239 /* WaKBLVECSSemaphoreWaitPoll:kbl */
1240 if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1241 wa_write(wal,
1242 RING_SEMA_WAIT_POLL(engine->mmio_base),
1243 1);
1244 }
1245}
1246
1247void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1248{
1249 struct i915_wa_list *wal = &engine->wa_list;
1250
1251 if (GEM_WARN_ON(INTEL_GEN(engine->i915) < 8))
1252 return;
1253
1254 wa_init_start(wal, engine->name);
1255
1256 if (engine->id == RCS)
1257 rcs_engine_wa_init(engine);
1258 else
1259 xcs_engine_wa_init(engine);
1260
1261 wa_init_finish(wal);
1262}
1263
1264void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1265{
1266 wa_list_apply(engine->i915, &engine->wa_list);
1267}
1268
1080#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1269#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1081#include "selftests/intel_workarounds.c" 1270#include "selftests/intel_workarounds.c"
1082#endif 1271#endif
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-28 08:57:22 -0400