aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm64/include/asm/cacheflush.h1
-rw-r--r--arch/arm64/mm/cache.S28
-rw-r--r--arch/arm64/mm/flush.c33
-rw-r--r--arch/arm64/mm/proc-macros.S22
4 files changed, 58 insertions, 26 deletions
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 54efedaf331f..7fc294c3bc5b 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -68,6 +68,7 @@
68extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); 68extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
69extern void flush_icache_range(unsigned long start, unsigned long end); 69extern void flush_icache_range(unsigned long start, unsigned long end);
70extern void __flush_dcache_area(void *addr, size_t len); 70extern void __flush_dcache_area(void *addr, size_t len);
71extern void __clean_dcache_area_pou(void *addr, size_t len);
71extern long __flush_cache_user_range(unsigned long start, unsigned long end); 72extern long __flush_cache_user_range(unsigned long start, unsigned long end);
72 73
73static inline void flush_cache_mm(struct mm_struct *mm) 74static inline void flush_cache_mm(struct mm_struct *mm)
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index cfa44a6adc0a..6df07069a025 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -81,26 +81,32 @@ ENDPROC(__flush_cache_user_range)
81/* 81/*
82 * __flush_dcache_area(kaddr, size) 82 * __flush_dcache_area(kaddr, size)
83 * 83 *
84 * Ensure that the data held in the page kaddr is written back to the 84 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
85 * page in question. 85 * are cleaned and invalidated to the PoC.
86 * 86 *
87 * - kaddr - kernel address 87 * - kaddr - kernel address
88 * - size - size in question 88 * - size - size in question
89 */ 89 */
90ENTRY(__flush_dcache_area) 90ENTRY(__flush_dcache_area)
91 dcache_line_size x2, x3 91 dcache_by_line_op civac, sy, x0, x1, x2, x3
92 add x1, x0, x1
93 sub x3, x2, #1
94 bic x0, x0, x3
951: dc civac, x0 // clean & invalidate D line / unified line
96 add x0, x0, x2
97 cmp x0, x1
98 b.lo 1b
99 dsb sy
100 ret 92 ret
101ENDPIPROC(__flush_dcache_area) 93ENDPIPROC(__flush_dcache_area)
102 94
103/* 95/*
96 * __clean_dcache_area_pou(kaddr, size)
97 *
98 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
99 * are cleaned to the PoU.
100 *
101 * - kaddr - kernel address
102 * - size - size in question
103 */
104ENTRY(__clean_dcache_area_pou)
105 dcache_by_line_op cvau, ish, x0, x1, x2, x3
106 ret
107ENDPROC(__clean_dcache_area_pou)
108
109/*
104 * __inval_cache_range(start, end) 110 * __inval_cache_range(start, end)
105 * - start - start address of region 111 * - start - start address of region
106 * - end - end address of region 112 * - end - end address of region
diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
index c26b804015e8..46649d6e6c5a 100644
--- a/arch/arm64/mm/flush.c
+++ b/arch/arm64/mm/flush.c
@@ -34,19 +34,24 @@ void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
34 __flush_icache_all(); 34 __flush_icache_all();
35} 35}
36 36
37static void sync_icache_aliases(void *kaddr, unsigned long len)
38{
39 unsigned long addr = (unsigned long)kaddr;
40
41 if (icache_is_aliasing()) {
42 __clean_dcache_area_pou(kaddr, len);
43 __flush_icache_all();
44 } else {
45 flush_icache_range(addr, addr + len);
46 }
47}
48
37static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, 49static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
38 unsigned long uaddr, void *kaddr, 50 unsigned long uaddr, void *kaddr,
39 unsigned long len) 51 unsigned long len)
40{ 52{
41 if (vma->vm_flags & VM_EXEC) { 53 if (vma->vm_flags & VM_EXEC)
42 unsigned long addr = (unsigned long)kaddr; 54 sync_icache_aliases(kaddr, len);
43 if (icache_is_aliasing()) {
44 __flush_dcache_area(kaddr, len);
45 __flush_icache_all();
46 } else {
47 flush_icache_range(addr, addr + len);
48 }
49 }
50} 55}
51 56
52/* 57/*
@@ -74,13 +79,11 @@ void __sync_icache_dcache(pte_t pte, unsigned long addr)
74 if (!page_mapping(page)) 79 if (!page_mapping(page))
75 return; 80 return;
76 81
77 if (!test_and_set_bit(PG_dcache_clean, &page->flags)) { 82 if (!test_and_set_bit(PG_dcache_clean, &page->flags))
78 __flush_dcache_area(page_address(page), 83 sync_icache_aliases(page_address(page),
79 PAGE_SIZE << compound_order(page)); 84 PAGE_SIZE << compound_order(page));
85 else if (icache_is_aivivt())
80 __flush_icache_all(); 86 __flush_icache_all();
81 } else if (icache_is_aivivt()) {
82 __flush_icache_all();
83 }
84} 87}
85 88
86/* 89/*
diff --git a/arch/arm64/mm/proc-macros.S b/arch/arm64/mm/proc-macros.S
index 4c4d93c4bf65..146bd99a7532 100644
--- a/arch/arm64/mm/proc-macros.S
+++ b/arch/arm64/mm/proc-macros.S
@@ -62,3 +62,25 @@
62 bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 62 bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
63#endif 63#endif
64 .endm 64 .endm
65
66/*
67 * Macro to perform a data cache maintenance for the interval
68 * [kaddr, kaddr + size)
69 *
70 * op: operation passed to dc instruction
71 * domain: domain used in dsb instruciton
72 * kaddr: starting virtual address of the region
73 * size: size of the region
74 * Corrupts: kaddr, size, tmp1, tmp2
75 */
76 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
77 dcache_line_size \tmp1, \tmp2
78 add \size, \kaddr, \size
79 sub \tmp2, \tmp1, #1
80 bic \kaddr, \kaddr, \tmp2
819998: dc \op, \kaddr
82 add \kaddr, \kaddr, \tmp1
83 cmp \kaddr, \size
84 b.lo 9998b
85 dsb \domain
86 .endm
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-28 19:24:39 -0500