7 files changed, 61 insertions, 1 deletions

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index dfd908630631..0b0576a54724 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -960,6 +960,17 @@ config ARM64_UAO
           regular load/store instructions if the cpu does not implement the
           feature.
 
+config ARM64_PMEM
+        bool "Enable support for persistent memory"
+        select ARCH_HAS_PMEM_API
+        help
+          Say Y to enable support for the persistent memory API based on the
+          ARMv8.2 DCPoP feature.
+
+          The feature is detected at runtime, and the kernel will use DC CVAC
+          operations if DC CVAP is not supported (following the behaviour of
+          DC CVAP itself if the system does not define a point of persistence).
+
 endmenu
 
 config ARM64_MODULE_CMODEL_LARGE
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 1b67c3782d00..5d8903c45031 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -353,6 +353,12 @@ alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
 alternative_else
         dc      civac, \kaddr
 alternative_endif
+        .elseif (\op == cvap)
+alternative_if ARM64_HAS_DCPOP
+        sys 3, c7, c12, 1, \kaddr       // dc cvap
+alternative_else
+        dc      cvac, \kaddr
+alternative_endif
         .else
         dc      \op, \kaddr
         .endif
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index b4b43a94dffd..76d1cc85d5b1 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -69,6 +69,7 @@ extern void flush_icache_range(unsigned long start, unsigned long end);
 extern void __flush_dcache_area(void *addr, size_t len);
 extern void __inval_dcache_area(void *addr, size_t len);
 extern void __clean_dcache_area_poc(void *addr, size_t len);
+extern void __clean_dcache_area_pop(void *addr, size_t len);
 extern void __clean_dcache_area_pou(void *addr, size_t len);
 extern long __flush_cache_user_range(unsigned long start, unsigned long end);
 extern void sync_icache_aliases(void *kaddr, unsigned long len);
diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 8d2272c6822c..8da621627d7c 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -39,7 +39,8 @@
 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003      18
 #define ARM64_WORKAROUND_858921                 19
 #define ARM64_WORKAROUND_CAVIUM_30115           20
+#define ARM64_HAS_DCPOP                         21
 
-#define ARM64_NCAPS                             21
+#define ARM64_NCAPS                             22
 
 #endif /* __ASM_CPUCAPS_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index a2542ef3ff25..cd52d365d1f0 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -889,6 +889,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                 .min_field_value = 0,
                 .matches = has_no_fpsimd,
         },
+#ifdef CONFIG_ARM64_PMEM
+        {
+                .desc = "Data cache clean to Point of Persistence",
+                .capability = ARM64_HAS_DCPOP,
+                .def_scope = SCOPE_SYSTEM,
+                .matches = has_cpuid_feature,
+                .sys_reg = SYS_ID_AA64ISAR1_EL1,
+                .field_pos = ID_AA64ISAR1_DPB_SHIFT,
+                .min_field_value = 1,
+        },
+#endif
         {},
 };
 
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index ed47fbbb4b05..7f1dbe962cf5 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -172,6 +172,20 @@ ENDPIPROC(__clean_dcache_area_poc)
 ENDPROC(__dma_clean_area)
 
 /*
+ *      __clean_dcache_area_pop(kaddr, size)
+ *
+ *      Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ *      are cleaned to the PoP.
+ *
+ *      - kaddr   - kernel address
+ *      - size    - size in question
+ */
+ENTRY(__clean_dcache_area_pop)
+        dcache_by_line_op cvap, sy, x0, x1, x2, x3
+        ret
+ENDPIPROC(__clean_dcache_area_pop)
+
+/*
  *      __dma_flush_area(start, size)
  *
  *      clean & invalidate D / U line
diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
index 21a8d828cbf4..280f90ff33a2 100644
--- a/arch/arm64/mm/flush.c
+++ b/arch/arm64/mm/flush.c
@@ -83,3 +83,19 @@ EXPORT_SYMBOL(flush_dcache_page);
  * Additional functions defined in assembly.
  */
 EXPORT_SYMBOL(flush_icache_range);
+
+#ifdef CONFIG_ARCH_HAS_PMEM_API
+static inline void arch_wb_cache_pmem(void *addr, size_t size)
+{
+        /* Ensure order against any prior non-cacheable writes */
+        dmb(osh);
+        __clean_dcache_area_pop(addr, size);
+}
+EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
+
+static inline void arch_invalidate_pmem(void *addr, size_t size)
+{
+        __inval_dcache_area(addr, size);
+}
+EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
+#endif