diff options
author | Catalin Marinas <catalin.marinas@arm.com> | 2012-03-05 06:49:28 -0500 |
---|---|---|
committer | Catalin Marinas <catalin.marinas@arm.com> | 2012-09-17 08:42:00 -0400 |
commit | f1a0c4aa0937975b53991842a494f741d7769b02 (patch) | |
tree | 9b344c5267cb982b14a2372a0a20714f5b36d61a | |
parent | 9cce7a435f89c9e60f244d44da2cf1cf4ed094ac (diff) |
arm64: Cache maintenance routines
The patch adds functionality required for cache maintenance. The AArch64
architecture mandates non-aliasing VIPT or PIPT D-cache and VIPT (may
have aliases) or ASID-tagged VIVT I-cache. Cache maintenance operations
are automatically broadcast in hardware between CPUs.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Tony Lindgren <tony@atomide.com>
Acked-by: Nicolas Pitre <nico@linaro.org>
Acked-by: Olof Johansson <olof@lixom.net>
Acked-by: Santosh Shilimkar <santosh.shilimkar@ti.com>
Acked-by: Arnd Bergmann <arnd@arndb.de>
-rw-r--r-- | arch/arm64/include/asm/cache.h | 32 | ||||
-rw-r--r-- | arch/arm64/include/asm/cacheflush.h | 148 | ||||
-rw-r--r-- | arch/arm64/include/asm/cachetype.h | 48 | ||||
-rw-r--r-- | arch/arm64/mm/cache.S | 168 | ||||
-rw-r--r-- | arch/arm64/mm/flush.c | 135 |
5 files changed, 531 insertions, 0 deletions
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h new file mode 100644 index 000000000000..390308a67f0d --- /dev/null +++ b/arch/arm64/include/asm/cache.h | |||
@@ -0,0 +1,32 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2012 ARM Ltd. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope that it will be useful, | ||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
11 | * GNU General Public License for more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | #ifndef __ASM_CACHE_H | ||
17 | #define __ASM_CACHE_H | ||
18 | |||
19 | #define L1_CACHE_SHIFT 6 | ||
20 | #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) | ||
21 | |||
22 | /* | ||
23 | * Memory returned by kmalloc() may be used for DMA, so we must make | ||
24 | * sure that all such allocations are cache aligned. Otherwise, | ||
25 | * unrelated code may cause parts of the buffer to be read into the | ||
26 | * cache before the transfer is done, causing old data to be seen by | ||
27 | * the CPU. | ||
28 | */ | ||
29 | #define ARCH_DMA_MINALIGN L1_CACHE_BYTES | ||
30 | #define ARCH_SLAB_MINALIGN 8 | ||
31 | |||
32 | #endif | ||
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h new file mode 100644 index 000000000000..aa3132ab7f29 --- /dev/null +++ b/arch/arm64/include/asm/cacheflush.h | |||
@@ -0,0 +1,148 @@ | |||
1 | /* | ||
2 | * Based on arch/arm/include/asm/cacheflush.h | ||
3 | * | ||
4 | * Copyright (C) 1999-2002 Russell King. | ||
5 | * Copyright (C) 2012 ARM Ltd. | ||
6 | * | ||
7 | * This program is free software; you can redistribute it and/or modify | ||
8 | * it under the terms of the GNU General Public License version 2 as | ||
9 | * published by the Free Software Foundation. | ||
10 | * | ||
11 | * This program is distributed in the hope that it will be useful, | ||
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | * GNU General Public License for more details. | ||
15 | * | ||
16 | * You should have received a copy of the GNU General Public License | ||
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
18 | */ | ||
19 | #ifndef __ASM_CACHEFLUSH_H | ||
20 | #define __ASM_CACHEFLUSH_H | ||
21 | |||
22 | #include <linux/mm.h> | ||
23 | |||
24 | /* | ||
25 | * This flag is used to indicate that the page pointed to by a pte is clean | ||
26 | * and does not require cleaning before returning it to the user. | ||
27 | */ | ||
28 | #define PG_dcache_clean PG_arch_1 | ||
29 | |||
30 | /* | ||
31 | * MM Cache Management | ||
32 | * =================== | ||
33 | * | ||
34 | * The arch/arm64/mm/cache.S implements these methods. | ||
35 | * | ||
36 | * Start addresses are inclusive and end addresses are exclusive; start | ||
37 | * addresses should be rounded down, end addresses up. | ||
38 | * | ||
39 | * See Documentation/cachetlb.txt for more information. Please note that | ||
40 | * the implementation assumes non-aliasing VIPT D-cache and (aliasing) | ||
41 | * VIPT or ASID-tagged VIVT I-cache. | ||
42 | * | ||
43 | * flush_cache_all() | ||
44 | * | ||
45 | * Unconditionally clean and invalidate the entire cache. | ||
46 | * | ||
47 | * flush_cache_mm(mm) | ||
48 | * | ||
49 | * Clean and invalidate all user space cache entries | ||
50 | * before a change of page tables. | ||
51 | * | ||
52 | * flush_icache_range(start, end) | ||
53 | * | ||
54 | * Ensure coherency between the I-cache and the D-cache in the | ||
55 | * region described by start, end. | ||
56 | * - start - virtual start address | ||
57 | * - end - virtual end address | ||
58 | * | ||
59 | * __flush_cache_user_range(start, end) | ||
60 | * | ||
61 | * Ensure coherency between the I-cache and the D-cache in the | ||
62 | * region described by start, end. | ||
63 | * - start - virtual start address | ||
64 | * - end - virtual end address | ||
65 | * | ||
66 | * __flush_dcache_area(kaddr, size) | ||
67 | * | ||
68 | * Ensure that the data held in page is written back. | ||
69 | * - kaddr - page address | ||
70 | * - size - region size | ||
71 | */ | ||
72 | extern void flush_cache_all(void); | ||
73 | extern void flush_cache_mm(struct mm_struct *mm); | ||
74 | extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); | ||
75 | extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn); | ||
76 | extern void flush_icache_range(unsigned long start, unsigned long end); | ||
77 | extern void __flush_dcache_area(void *addr, size_t len); | ||
78 | extern void __flush_cache_user_range(unsigned long start, unsigned long end); | ||
79 | |||
80 | /* | ||
81 | * Copy user data from/to a page which is mapped into a different | ||
82 | * processes address space. Really, we want to allow our "user | ||
83 | * space" model to handle this. | ||
84 | */ | ||
85 | extern void copy_to_user_page(struct vm_area_struct *, struct page *, | ||
86 | unsigned long, void *, const void *, unsigned long); | ||
87 | #define copy_from_user_page(vma, page, vaddr, dst, src, len) \ | ||
88 | do { \ | ||
89 | memcpy(dst, src, len); \ | ||
90 | } while (0) | ||
91 | |||
92 | #define flush_cache_dup_mm(mm) flush_cache_mm(mm) | ||
93 | |||
94 | /* | ||
95 | * flush_dcache_page is used when the kernel has written to the page | ||
96 | * cache page at virtual address page->virtual. | ||
97 | * | ||
98 | * If this page isn't mapped (ie, page_mapping == NULL), or it might | ||
99 | * have userspace mappings, then we _must_ always clean + invalidate | ||
100 | * the dcache entries associated with the kernel mapping. | ||
101 | * | ||
102 | * Otherwise we can defer the operation, and clean the cache when we are | ||
103 | * about to change to user space. This is the same method as used on SPARC64. | ||
104 | * See update_mmu_cache for the user space part. | ||
105 | */ | ||
106 | #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 | ||
107 | extern void flush_dcache_page(struct page *); | ||
108 | |||
109 | static inline void __flush_icache_all(void) | ||
110 | { | ||
111 | asm("ic ialluis"); | ||
112 | } | ||
113 | |||
114 | #define flush_dcache_mmap_lock(mapping) \ | ||
115 | spin_lock_irq(&(mapping)->tree_lock) | ||
116 | #define flush_dcache_mmap_unlock(mapping) \ | ||
117 | spin_unlock_irq(&(mapping)->tree_lock) | ||
118 | |||
119 | #define flush_icache_user_range(vma,page,addr,len) \ | ||
120 | flush_dcache_page(page) | ||
121 | |||
122 | /* | ||
123 | * We don't appear to need to do anything here. In fact, if we did, we'd | ||
124 | * duplicate cache flushing elsewhere performed by flush_dcache_page(). | ||
125 | */ | ||
126 | #define flush_icache_page(vma,page) do { } while (0) | ||
127 | |||
128 | /* | ||
129 | * flush_cache_vmap() is used when creating mappings (eg, via vmap, | ||
130 | * vmalloc, ioremap etc) in kernel space for pages. On non-VIPT | ||
131 | * caches, since the direct-mappings of these pages may contain cached | ||
132 | * data, we need to do a full cache flush to ensure that writebacks | ||
133 | * don't corrupt data placed into these pages via the new mappings. | ||
134 | */ | ||
135 | static inline void flush_cache_vmap(unsigned long start, unsigned long end) | ||
136 | { | ||
137 | /* | ||
138 | * set_pte_at() called from vmap_pte_range() does not | ||
139 | * have a DSB after cleaning the cache line. | ||
140 | */ | ||
141 | dsb(); | ||
142 | } | ||
143 | |||
144 | static inline void flush_cache_vunmap(unsigned long start, unsigned long end) | ||
145 | { | ||
146 | } | ||
147 | |||
148 | #endif | ||
diff --git a/arch/arm64/include/asm/cachetype.h b/arch/arm64/include/asm/cachetype.h new file mode 100644 index 000000000000..85f5f511352a --- /dev/null +++ b/arch/arm64/include/asm/cachetype.h | |||
@@ -0,0 +1,48 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2012 ARM Ltd. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope that it will be useful, | ||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
11 | * GNU General Public License for more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | #ifndef __ASM_CACHETYPE_H | ||
17 | #define __ASM_CACHETYPE_H | ||
18 | |||
19 | #include <asm/cputype.h> | ||
20 | |||
21 | #define CTR_L1IP_SHIFT 14 | ||
22 | #define CTR_L1IP_MASK 3 | ||
23 | |||
24 | #define ICACHE_POLICY_RESERVED 0 | ||
25 | #define ICACHE_POLICY_AIVIVT 1 | ||
26 | #define ICACHE_POLICY_VIPT 2 | ||
27 | #define ICACHE_POLICY_PIPT 3 | ||
28 | |||
29 | static inline u32 icache_policy(void) | ||
30 | { | ||
31 | return (read_cpuid_cachetype() >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK; | ||
32 | } | ||
33 | |||
34 | /* | ||
35 | * Whilst the D-side always behaves as PIPT on AArch64, aliasing is | ||
36 | * permitted in the I-cache. | ||
37 | */ | ||
38 | static inline int icache_is_aliasing(void) | ||
39 | { | ||
40 | return icache_policy() != ICACHE_POLICY_PIPT; | ||
41 | } | ||
42 | |||
43 | static inline int icache_is_aivivt(void) | ||
44 | { | ||
45 | return icache_policy() == ICACHE_POLICY_AIVIVT; | ||
46 | } | ||
47 | |||
48 | #endif /* __ASM_CACHETYPE_H */ | ||
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S new file mode 100644 index 000000000000..abe69b80cf7f --- /dev/null +++ b/arch/arm64/mm/cache.S | |||
@@ -0,0 +1,168 @@ | |||
1 | /* | ||
2 | * Cache maintenance | ||
3 | * | ||
4 | * Copyright (C) 2001 Deep Blue Solutions Ltd. | ||
5 | * Copyright (C) 2012 ARM Ltd. | ||
6 | * | ||
7 | * This program is free software; you can redistribute it and/or modify | ||
8 | * it under the terms of the GNU General Public License version 2 as | ||
9 | * published by the Free Software Foundation. | ||
10 | * | ||
11 | * This program is distributed in the hope that it will be useful, | ||
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | * GNU General Public License for more details. | ||
15 | * | ||
16 | * You should have received a copy of the GNU General Public License | ||
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
18 | */ | ||
19 | |||
20 | #include <linux/linkage.h> | ||
21 | #include <linux/init.h> | ||
22 | #include <asm/assembler.h> | ||
23 | |||
24 | #include "proc-macros.S" | ||
25 | |||
26 | /* | ||
27 | * __flush_dcache_all() | ||
28 | * | ||
29 | * Flush the whole D-cache. | ||
30 | * | ||
31 | * Corrupted registers: x0-x7, x9-x11 | ||
32 | */ | ||
33 | ENTRY(__flush_dcache_all) | ||
34 | dsb sy // ensure ordering with previous memory accesses | ||
35 | mrs x0, clidr_el1 // read clidr | ||
36 | and x3, x0, #0x7000000 // extract loc from clidr | ||
37 | lsr x3, x3, #23 // left align loc bit field | ||
38 | cbz x3, finished // if loc is 0, then no need to clean | ||
39 | mov x10, #0 // start clean at cache level 0 | ||
40 | loop1: | ||
41 | add x2, x10, x10, lsr #1 // work out 3x current cache level | ||
42 | lsr x1, x0, x2 // extract cache type bits from clidr | ||
43 | and x1, x1, #7 // mask of the bits for current cache only | ||
44 | cmp x1, #2 // see what cache we have at this level | ||
45 | b.lt skip // skip if no cache, or just i-cache | ||
46 | save_and_disable_irqs x9 // make CSSELR and CCSIDR access atomic | ||
47 | msr csselr_el1, x10 // select current cache level in csselr | ||
48 | isb // isb to sych the new cssr&csidr | ||
49 | mrs x1, ccsidr_el1 // read the new ccsidr | ||
50 | restore_irqs x9 | ||
51 | and x2, x1, #7 // extract the length of the cache lines | ||
52 | add x2, x2, #4 // add 4 (line length offset) | ||
53 | mov x4, #0x3ff | ||
54 | and x4, x4, x1, lsr #3 // find maximum number on the way size | ||
55 | clz x5, x4 // find bit position of way size increment | ||
56 | mov x7, #0x7fff | ||
57 | and x7, x7, x1, lsr #13 // extract max number of the index size | ||
58 | loop2: | ||
59 | mov x9, x4 // create working copy of max way size | ||
60 | loop3: | ||
61 | lsl x6, x9, x5 | ||
62 | orr x11, x10, x6 // factor way and cache number into x11 | ||
63 | lsl x6, x7, x2 | ||
64 | orr x11, x11, x6 // factor index number into x11 | ||
65 | dc cisw, x11 // clean & invalidate by set/way | ||
66 | subs x9, x9, #1 // decrement the way | ||
67 | b.ge loop3 | ||
68 | subs x7, x7, #1 // decrement the index | ||
69 | b.ge loop2 | ||
70 | skip: | ||
71 | add x10, x10, #2 // increment cache number | ||
72 | cmp x3, x10 | ||
73 | b.gt loop1 | ||
74 | finished: | ||
75 | mov x10, #0 // swith back to cache level 0 | ||
76 | msr csselr_el1, x10 // select current cache level in csselr | ||
77 | dsb sy | ||
78 | isb | ||
79 | ret | ||
80 | ENDPROC(__flush_dcache_all) | ||
81 | |||
82 | /* | ||
83 | * flush_cache_all() | ||
84 | * | ||
85 | * Flush the entire cache system. The data cache flush is now achieved | ||
86 | * using atomic clean / invalidates working outwards from L1 cache. This | ||
87 | * is done using Set/Way based cache maintainance instructions. The | ||
88 | * instruction cache can still be invalidated back to the point of | ||
89 | * unification in a single instruction. | ||
90 | */ | ||
91 | ENTRY(flush_cache_all) | ||
92 | mov x12, lr | ||
93 | bl __flush_dcache_all | ||
94 | mov x0, #0 | ||
95 | ic ialluis // I+BTB cache invalidate | ||
96 | ret x12 | ||
97 | ENDPROC(flush_cache_all) | ||
98 | |||
99 | /* | ||
100 | * flush_icache_range(start,end) | ||
101 | * | ||
102 | * Ensure that the I and D caches are coherent within specified region. | ||
103 | * This is typically used when code has been written to a memory region, | ||
104 | * and will be executed. | ||
105 | * | ||
106 | * - start - virtual start address of region | ||
107 | * - end - virtual end address of region | ||
108 | */ | ||
109 | ENTRY(flush_icache_range) | ||
110 | /* FALLTHROUGH */ | ||
111 | |||
112 | /* | ||
113 | * __flush_cache_user_range(start,end) | ||
114 | * | ||
115 | * Ensure that the I and D caches are coherent within specified region. | ||
116 | * This is typically used when code has been written to a memory region, | ||
117 | * and will be executed. | ||
118 | * | ||
119 | * - start - virtual start address of region | ||
120 | * - end - virtual end address of region | ||
121 | */ | ||
122 | ENTRY(__flush_cache_user_range) | ||
123 | dcache_line_size x2, x3 | ||
124 | sub x3, x2, #1 | ||
125 | bic x4, x0, x3 | ||
126 | 1: | ||
127 | USER(9f, dc cvau, x4 ) // clean D line to PoU | ||
128 | add x4, x4, x2 | ||
129 | cmp x4, x1 | ||
130 | b.lo 1b | ||
131 | dsb sy | ||
132 | |||
133 | icache_line_size x2, x3 | ||
134 | sub x3, x2, #1 | ||
135 | bic x4, x0, x3 | ||
136 | 1: | ||
137 | USER(9f, ic ivau, x4 ) // invalidate I line PoU | ||
138 | add x4, x4, x2 | ||
139 | cmp x4, x1 | ||
140 | b.lo 1b | ||
141 | 9: // ignore any faulting cache operation | ||
142 | dsb sy | ||
143 | isb | ||
144 | ret | ||
145 | ENDPROC(flush_icache_range) | ||
146 | ENDPROC(__flush_cache_user_range) | ||
147 | |||
148 | /* | ||
149 | * __flush_kern_dcache_page(kaddr) | ||
150 | * | ||
151 | * Ensure that the data held in the page kaddr is written back to the | ||
152 | * page in question. | ||
153 | * | ||
154 | * - kaddr - kernel address | ||
155 | * - size - size in question | ||
156 | */ | ||
157 | ENTRY(__flush_dcache_area) | ||
158 | dcache_line_size x2, x3 | ||
159 | add x1, x0, x1 | ||
160 | sub x3, x2, #1 | ||
161 | bic x0, x0, x3 | ||
162 | 1: dc civac, x0 // clean & invalidate D line / unified line | ||
163 | add x0, x0, x2 | ||
164 | cmp x0, x1 | ||
165 | b.lo 1b | ||
166 | dsb sy | ||
167 | ret | ||
168 | ENDPROC(__flush_dcache_area) | ||
diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c new file mode 100644 index 000000000000..c144adb1682f --- /dev/null +++ b/arch/arm64/mm/flush.c | |||
@@ -0,0 +1,135 @@ | |||
1 | /* | ||
2 | * Based on arch/arm/mm/flush.c | ||
3 | * | ||
4 | * Copyright (C) 1995-2002 Russell King | ||
5 | * Copyright (C) 2012 ARM Ltd. | ||
6 | * | ||
7 | * This program is free software; you can redistribute it and/or modify | ||
8 | * it under the terms of the GNU General Public License version 2 as | ||
9 | * published by the Free Software Foundation. | ||
10 | * | ||
11 | * This program is distributed in the hope that it will be useful, | ||
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | * GNU General Public License for more details. | ||
15 | * | ||
16 | * You should have received a copy of the GNU General Public License | ||
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
18 | */ | ||
19 | |||
20 | #include <linux/export.h> | ||
21 | #include <linux/mm.h> | ||
22 | #include <linux/pagemap.h> | ||
23 | |||
24 | #include <asm/cacheflush.h> | ||
25 | #include <asm/cachetype.h> | ||
26 | #include <asm/tlbflush.h> | ||
27 | |||
28 | #include "mm.h" | ||
29 | |||
30 | void flush_cache_mm(struct mm_struct *mm) | ||
31 | { | ||
32 | } | ||
33 | |||
34 | void flush_cache_range(struct vm_area_struct *vma, unsigned long start, | ||
35 | unsigned long end) | ||
36 | { | ||
37 | if (vma->vm_flags & VM_EXEC) | ||
38 | __flush_icache_all(); | ||
39 | } | ||
40 | |||
41 | void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, | ||
42 | unsigned long pfn) | ||
43 | { | ||
44 | } | ||
45 | |||
46 | static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, | ||
47 | unsigned long uaddr, void *kaddr, | ||
48 | unsigned long len) | ||
49 | { | ||
50 | if (vma->vm_flags & VM_EXEC) { | ||
51 | unsigned long addr = (unsigned long)kaddr; | ||
52 | if (icache_is_aliasing()) { | ||
53 | __flush_dcache_area(kaddr, len); | ||
54 | __flush_icache_all(); | ||
55 | } else { | ||
56 | flush_icache_range(addr, addr + len); | ||
57 | } | ||
58 | } | ||
59 | } | ||
60 | |||
61 | /* | ||
62 | * Copy user data from/to a page which is mapped into a different processes | ||
63 | * address space. Really, we want to allow our "user space" model to handle | ||
64 | * this. | ||
65 | * | ||
66 | * Note that this code needs to run on the current CPU. | ||
67 | */ | ||
68 | void copy_to_user_page(struct vm_area_struct *vma, struct page *page, | ||
69 | unsigned long uaddr, void *dst, const void *src, | ||
70 | unsigned long len) | ||
71 | { | ||
72 | #ifdef CONFIG_SMP | ||
73 | preempt_disable(); | ||
74 | #endif | ||
75 | memcpy(dst, src, len); | ||
76 | flush_ptrace_access(vma, page, uaddr, dst, len); | ||
77 | #ifdef CONFIG_SMP | ||
78 | preempt_enable(); | ||
79 | #endif | ||
80 | } | ||
81 | |||
82 | void __flush_dcache_page(struct page *page) | ||
83 | { | ||
84 | __flush_dcache_area(page_address(page), PAGE_SIZE); | ||
85 | } | ||
86 | |||
87 | void __sync_icache_dcache(pte_t pte, unsigned long addr) | ||
88 | { | ||
89 | unsigned long pfn; | ||
90 | struct page *page; | ||
91 | |||
92 | pfn = pte_pfn(pte); | ||
93 | if (!pfn_valid(pfn)) | ||
94 | return; | ||
95 | |||
96 | page = pfn_to_page(pfn); | ||
97 | if (!test_and_set_bit(PG_dcache_clean, &page->flags)) { | ||
98 | __flush_dcache_page(page); | ||
99 | __flush_icache_all(); | ||
100 | } else if (icache_is_aivivt()) { | ||
101 | __flush_icache_all(); | ||
102 | } | ||
103 | } | ||
104 | |||
105 | /* | ||
106 | * Ensure cache coherency between kernel mapping and userspace mapping of this | ||
107 | * page. | ||
108 | */ | ||
109 | void flush_dcache_page(struct page *page) | ||
110 | { | ||
111 | struct address_space *mapping; | ||
112 | |||
113 | /* | ||
114 | * The zero page is never written to, so never has any dirty cache | ||
115 | * lines, and therefore never needs to be flushed. | ||
116 | */ | ||
117 | if (page == ZERO_PAGE(0)) | ||
118 | return; | ||
119 | |||
120 | mapping = page_mapping(page); | ||
121 | if (mapping && mapping_mapped(mapping)) { | ||
122 | __flush_dcache_page(page); | ||
123 | __flush_icache_all(); | ||
124 | set_bit(PG_dcache_clean, &page->flags); | ||
125 | } else { | ||
126 | clear_bit(PG_dcache_clean, &page->flags); | ||
127 | } | ||
128 | } | ||
129 | EXPORT_SYMBOL(flush_dcache_page); | ||
130 | |||
131 | /* | ||
132 | * Additional functions defined in assembly. | ||
133 | */ | ||
134 | EXPORT_SYMBOL(flush_cache_all); | ||
135 | EXPORT_SYMBOL(flush_icache_range); | ||