5 files changed, 60 insertions, 8 deletions

diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index b2e6ece23713..5df5cfe1c143 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -20,8 +20,12 @@
 
 #define CTR_L1IP_SHIFT          14
 #define CTR_L1IP_MASK           3
+#define CTR_DMINLINE_SHIFT      16
+#define CTR_ERG_SHIFT           20
 #define CTR_CWG_SHIFT           24
 #define CTR_CWG_MASK            15
+#define CTR_IDC_SHIFT           28
+#define CTR_DIC_SHIFT           29
 
 #define CTR_L1IP(ctr)           (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
 
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index bef9f418f089..7dfcec4700fe 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -133,6 +133,9 @@ extern void flush_dcache_page(struct page *);
 
 static inline void __flush_icache_all(void)
 {
+        if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
+                return;
+
         asm("ic ialluis");
         dsb(ish);
 }
diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 39134c46bb13..ff9fb3aba17b 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -46,7 +46,9 @@
 #define ARM64_HARDEN_BP_POST_GUEST_EXIT         25
 #define ARM64_HAS_RAS_EXTN                      26
 #define ARM64_WORKAROUND_843419                 27
+#define ARM64_HAS_CACHE_IDC                     28
+#define ARM64_HAS_CACHE_DIC                     29
 
-#define ARM64_NCAPS                             28
+#define ARM64_NCAPS                             30
 
 #endif /* __ASM_CPUCAPS_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index f96b3449034b..bdab55223866 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -199,12 +199,12 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
 };
 
 static const struct arm64_ftr_bits ftr_ctr[] = {
-        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1),           /* RES1 */
-        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 29, 1, 1),      /* DIC */
-        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 28, 1, 1),      /* IDC */
-        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0),     /* CWG */
-        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 20, 4, 0),     /* ERG */
-        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1),      /* DminLine */
+        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
+        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DIC_SHIFT, 1, 1),
+        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IDC_SHIFT, 1, 1),
+        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_CWG_SHIFT, 4, 0),
+        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_ERG_SHIFT, 4, 0),
+        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DMINLINE_SHIFT, 4, 1),
         /*
          * Linux can handle differing I-cache policies. Userspace JITs will
          * make use of *minLine.
@@ -852,6 +852,18 @@ static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unus
                                         ID_AA64PFR0_FP_SHIFT) < 0;
 }
 
+static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
+                          int __unused)
+{
+        return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_IDC_SHIFT);
+}
+
+static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
+                          int __unused)
+{
+        return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_DIC_SHIFT);
+}
+
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
 static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
 
@@ -1088,6 +1100,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                 .enable = cpu_clear_disr,
         },
 #endif /* CONFIG_ARM64_RAS_EXTN */
+        {
+                .desc = "Data cache clean to the PoU not required for I/D coherence",
+                .capability = ARM64_HAS_CACHE_IDC,
+                .def_scope = SCOPE_SYSTEM,
+                .matches = has_cache_idc,
+        },
+        {
+                .desc = "Instruction cache invalidation not required for I/D coherence",
+                .capability = ARM64_HAS_CACHE_DIC,
+                .def_scope = SCOPE_SYSTEM,
+                .matches = has_cache_dic,
+        },
         {},
 };
 
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 758bde7e2fa6..30334d81b021 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -50,6 +50,10 @@ ENTRY(flush_icache_range)
  */
 ENTRY(__flush_cache_user_range)
         uaccess_ttbr0_enable x2, x3, x4
+alternative_if ARM64_HAS_CACHE_IDC
+        dsb     ishst
+        b       7f
+alternative_else_nop_endif
         dcache_line_size x2, x3
         sub     x3, x2, #1
         bic     x4, x0, x3
@@ -60,8 +64,13 @@ user_alt 9f, "dc cvau, x4",  "dc civac, x4",  ARM64_WORKAROUND_CLEAN_CACHE
         b.lo    1b
         dsb     ish
 
+7:
+alternative_if ARM64_HAS_CACHE_DIC
+        isb
+        b       8f
+alternative_else_nop_endif
         invalidate_icache_by_line x0, x1, x2, x3, 9f
-        mov     x0, #0
+8:      mov     x0, #0
 1:
         uaccess_ttbr0_disable x1, x2
         ret
@@ -80,6 +89,12 @@ ENDPROC(__flush_cache_user_range)
  *      - end     - virtual end address of region
  */
 ENTRY(invalidate_icache_range)
+alternative_if ARM64_HAS_CACHE_DIC
+        mov     x0, xzr
+        isb
+        ret
+alternative_else_nop_endif
+
         uaccess_ttbr0_enable x2, x3, x4
 
         invalidate_icache_by_line x0, x1, x2, x3, 2f
@@ -116,6 +131,10 @@ ENDPIPROC(__flush_dcache_area)
  *      - size    - size in question
  */
 ENTRY(__clean_dcache_area_pou)
+alternative_if ARM64_HAS_CACHE_IDC
+        dsb     ishst
+        ret
+alternative_else_nop_endif
         dcache_by_line_op cvau, ish, x0, x1, x2, x3
         ret
 ENDPROC(__clean_dcache_area_pou)