arm64: add Cortex-A57 erratum 832075 workaround

The ARM erratum 832075 applies to certain revisions of Cortex-A57,
one of the workarounds is to change device loads into using
load-aquire semantics.
This is achieved using the alternatives framework.

Signed-off-by: Andre Przywara <andre.przywara@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>

3 files changed, 29 insertions, 6 deletions

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 92b6ee44669b..0362f8020d46 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -21,9 +21,10 @@
 #define MAX_CPU_FEATURES        (8 * sizeof(elf_hwcap))
 #define cpu_feature(x)          ilog2(HWCAP_ ## x)
 
-#define ARM64_WORKAROUND_CLEAN_CACHE    0
+#define ARM64_WORKAROUND_CLEAN_CACHE            0
+#define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE    1
 
-#define NCAPS                           1
+#define NCAPS                                   2
 
 #ifndef __ASSEMBLY__
 
diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 79f1d519221f..75825b63464d 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -28,6 +28,8 @@
 #include <asm/barrier.h>
 #include <asm/pgtable.h>
 #include <asm/early_ioremap.h>
+#include <asm/alternative.h>
+#include <asm/cpufeature.h>
 
 #include <xen/xen.h>
 
@@ -57,28 +59,41 @@ static inline void __raw_writeq(u64 val, volatile void __iomem *addr)
 static inline u8 __raw_readb(const volatile void __iomem *addr)
 {
         u8 val;
-        asm volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr));
+        asm volatile(ALTERNATIVE("ldrb %w0, [%1]",
+                                 "ldarb %w0, [%1]",
+                                 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+                     : "=r" (val) : "r" (addr));
         return val;
 }
 
 static inline u16 __raw_readw(const volatile void __iomem *addr)
 {
         u16 val;
-        asm volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr));
+
+        asm volatile(ALTERNATIVE("ldrh %w0, [%1]",
+                                 "ldarh %w0, [%1]",
+                                 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+                     : "=r" (val) : "r" (addr));
         return val;
 }
 
 static inline u32 __raw_readl(const volatile void __iomem *addr)
 {
         u32 val;
-        asm volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr));
+        asm volatile(ALTERNATIVE("ldr %w0, [%1]",
+                                 "ldar %w0, [%1]",
+                                 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+                     : "=r" (val) : "r" (addr));
         return val;
 }
 
 static inline u64 __raw_readq(const volatile void __iomem *addr)
 {
         u64 val;
-        asm volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr));
+        asm volatile(ALTERNATIVE("ldr %0, [%1]",
+                                 "ldar %0, [%1]",
+                                 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+                     : "=r" (val) : "r" (addr));
         return val;
 }
 
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index e107ed2d66dc..30935d2da55a 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -24,6 +24,7 @@
 #include <asm/cpufeature.h>
 
 #define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
+#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
 
 /*
  * Add a struct or another datatype to the union below if you need
@@ -71,6 +72,12 @@ struct arm64_cpu_capabilities arm64_errata[] = {
                 MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02),
         },
         {
+        /* Cortex-A57 r0p0 - r1p2 */
+                .desc = "ARM erratum 832075",
+                .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
+                MIDR_RANGE(MIDR_CORTEX_A57, 0x00, 0x12),
+        },
+        {
         }
 };