arm64: Add workaround for Cortex-A76 erratum 1286807

On the affected Cortex-A76 cores (r0p0 to r3p0), if a virtual address
for a cacheable mapping of a location is being accessed by a core while
another core is remapping the virtual address to a new physical page
using the recommended break-before-make sequence, then under very rare
circumstances TLBI+DSB completes before a read using the translation
being invalidated has been observed by other observers. The workaround
repeats the TLBI+DSB operation and is shared with the Qualcomm Falkor
erratum 1009

Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

4 files changed, 45 insertions, 5 deletions

diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt
index 76ccded8b74c..8f9577621144 100644
--- a/Documentation/arm64/silicon-errata.txt
+++ b/Documentation/arm64/silicon-errata.txt
@@ -57,6 +57,7 @@ stable kernels.
 | ARM            | Cortex-A73      | #858921         | ARM64_ERRATUM_858921        |
 | ARM            | Cortex-A55      | #1024718        | ARM64_ERRATUM_1024718       |
 | ARM            | Cortex-A76      | #1188873        | ARM64_ERRATUM_1188873       |
+| ARM            | Cortex-A76      | #1286807        | ARM64_ERRATUM_1286807       |
 | ARM            | MMU-500         | #841119,#826419 | N/A                         |
 |                |                 |                 |                             |
 | Cavium         | ThunderX ITS    | #22375, #24313  | CAVIUM_ERRATUM_22375        |
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 787d7850e064..ea2ab0330e3a 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -497,6 +497,24 @@ config ARM64_ERRATUM_1188873
 
           If unsure, say Y.
 
+config ARM64_ERRATUM_1286807
+        bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation"
+        default y
+        select ARM64_WORKAROUND_REPEAT_TLBI
+        help
+          This option adds workaround for ARM Cortex-A76 erratum 1286807
+
+          On the affected Cortex-A76 cores (r0p0 to r3p0), if a virtual
+          address for a cacheable mapping of a location is being
+          accessed by a core while another core is remapping the virtual
+          address to a new physical page using the recommended
+          break-before-make sequence, then under very rare circumstances
+          TLBI+DSB completes before a read using the translation being
+          invalidated has been observed by other observers. The
+          workaround repeats the TLBI+DSB operation.
+
+          If unsure, say Y.
+
 config CAVIUM_ERRATUM_22375
         bool "Cavium erratum 22375, 24313"
         default y
@@ -566,9 +584,16 @@ config QCOM_FALKOR_ERRATUM_1003
           is unchanged. Work around the erratum by invalidating the walk cache
           entries for the trampoline before entering the kernel proper.
 
+config ARM64_WORKAROUND_REPEAT_TLBI
+        bool
+        help
+          Enable the repeat TLBI workaround for Falkor erratum 1009 and
+          Cortex-A76 erratum 1286807.
+
 config QCOM_FALKOR_ERRATUM_1009
         bool "Falkor E1009: Prematurely complete a DSB after a TLBI"
         default y
+        select ARM64_WORKAROUND_REPEAT_TLBI
         help
           On Falkor v1, the CPU may prematurely complete a DSB following a
           TLBI xxIS invalidate maintenance operation. Repeat the TLBI operation
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index c3c0387aee18..5dfd23897dea 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -41,14 +41,14 @@
                    ALTERNATIVE("nop\n                   nop",                  \
                                "dsb ish\n               tlbi " #op,            \
                                ARM64_WORKAROUND_REPEAT_TLBI,                   \
-                               CONFIG_QCOM_FALKOR_ERRATUM_1009)                \
+                               CONFIG_ARM64_WORKAROUND_REPEAT_TLBI)            \
                             : : )
 
 #define __TLBI_1(op, arg) asm ("tlbi " #op ", %0\n"                            \
                    ALTERNATIVE("nop\n                   nop",                  \
                                "dsb ish\n               tlbi " #op ", %0",     \
                                ARM64_WORKAROUND_REPEAT_TLBI,                   \
-                               CONFIG_QCOM_FALKOR_ERRATUM_1009)                \
+                               CONFIG_ARM64_WORKAROUND_REPEAT_TLBI)            \
                             : : "r" (arg))
 
 #define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg)
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index a509e35132d2..6ad715d67df8 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -570,6 +570,20 @@ static const struct midr_range arm64_harden_el2_vectors[] = {
 
 #endif
 
+#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
+
+static const struct midr_range arm64_repeat_tlbi_cpus[] = {
+#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
+        MIDR_RANGE(MIDR_QCOM_FALKOR_V1, 0, 0, 0, 0),
+#endif
+#ifdef CONFIG_ARM64_ERRATUM_1286807
+        MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 0),
+#endif
+        {},
+};
+
+#endif
+
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
         defined(CONFIG_ARM64_ERRATUM_827319) || \
@@ -695,11 +709,11 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                 .matches = is_kryo_midr,
         },
 #endif
-#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
+#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
         {
-                .desc = "Qualcomm Technologies Falkor erratum 1009",
+                .desc = "Qualcomm erratum 1009, ARM erratum 1286807",
                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
-                ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
+                ERRATA_MIDR_RANGE_LIST(arm64_repeat_tlbi_cpus),
         },
 #endif
 #ifdef CONFIG_ARM64_ERRATUM_858921