aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sh/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sh/mm')
-rw-r--r--arch/sh/mm/Kconfig222
-rw-r--r--arch/sh/mm/Makefile40
-rw-r--r--arch/sh/mm/Makefile_3236
-rw-r--r--arch/sh/mm/Makefile_6444
-rw-r--r--arch/sh/mm/cache-debugfs.c9
-rw-r--r--arch/sh/mm/cache-sh4.c14
-rw-r--r--arch/sh/mm/cache-sh5.c1029
-rw-r--r--arch/sh/mm/cache-sh7705.c12
-rw-r--r--arch/sh/mm/clear_page.S152
-rw-r--r--arch/sh/mm/consistent.c174
-rw-r--r--arch/sh/mm/copy_page.S388
-rw-r--r--arch/sh/mm/extable_32.c (renamed from arch/sh/mm/extable.c)0
-rw-r--r--arch/sh/mm/extable_64.c82
-rw-r--r--arch/sh/mm/fault_32.c (renamed from arch/sh/mm/fault.c)2
-rw-r--r--arch/sh/mm/fault_64.c275
-rw-r--r--arch/sh/mm/init.c68
-rw-r--r--arch/sh/mm/ioremap_32.c (renamed from arch/sh/mm/ioremap.c)0
-rw-r--r--arch/sh/mm/ioremap_64.c404
-rw-r--r--arch/sh/mm/pg-nommu.c4
-rw-r--r--arch/sh/mm/pmb.c26
-rw-r--r--arch/sh/mm/tlb-nommu.c10
-rw-r--r--arch/sh/mm/tlb-sh4.c7
-rw-r--r--arch/sh/mm/tlb-sh5.c164
-rw-r--r--arch/sh/mm/tlbflush_32.c (renamed from arch/sh/mm/tlb-flush.c)0
-rw-r--r--arch/sh/mm/tlbflush_64.c475
25 files changed, 2774 insertions, 863 deletions
diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig
index 1265f204f7d1..f549b8cd2501 100644
--- a/arch/sh/mm/Kconfig
+++ b/arch/sh/mm/Kconfig
@@ -1,193 +1,3 @@
1#
2# Processor families
3#
4config CPU_SH2
5 bool
6
7config CPU_SH2A
8 bool
9 select CPU_SH2
10
11config CPU_SH3
12 bool
13 select CPU_HAS_INTEVT
14 select CPU_HAS_SR_RB
15
16config CPU_SH4
17 bool
18 select CPU_HAS_INTEVT
19 select CPU_HAS_SR_RB
20 select CPU_HAS_PTEA if !CPU_SH4A || CPU_SHX2
21 select CPU_HAS_FPU if !CPU_SH4AL_DSP
22
23config CPU_SH4A
24 bool
25 select CPU_SH4
26
27config CPU_SH4AL_DSP
28 bool
29 select CPU_SH4A
30 select CPU_HAS_DSP
31
32config CPU_SHX2
33 bool
34
35config CPU_SHX3
36 bool
37
38choice
39 prompt "Processor sub-type selection"
40
41#
42# Processor subtypes
43#
44
45# SH-2 Processor Support
46
47config CPU_SUBTYPE_SH7619
48 bool "Support SH7619 processor"
49 select CPU_SH2
50
51# SH-2A Processor Support
52
53config CPU_SUBTYPE_SH7206
54 bool "Support SH7206 processor"
55 select CPU_SH2A
56
57# SH-3 Processor Support
58
59config CPU_SUBTYPE_SH7705
60 bool "Support SH7705 processor"
61 select CPU_SH3
62
63config CPU_SUBTYPE_SH7706
64 bool "Support SH7706 processor"
65 select CPU_SH3
66 help
67 Select SH7706 if you have a 133 Mhz SH-3 HD6417706 CPU.
68
69config CPU_SUBTYPE_SH7707
70 bool "Support SH7707 processor"
71 select CPU_SH3
72 help
73 Select SH7707 if you have a 60 Mhz SH-3 HD6417707 CPU.
74
75config CPU_SUBTYPE_SH7708
76 bool "Support SH7708 processor"
77 select CPU_SH3
78 help
79 Select SH7708 if you have a 60 Mhz SH-3 HD6417708S or
80 if you have a 100 Mhz SH-3 HD6417708R CPU.
81
82config CPU_SUBTYPE_SH7709
83 bool "Support SH7709 processor"
84 select CPU_SH3
85 help
86 Select SH7709 if you have a 80 Mhz SH-3 HD6417709 CPU.
87
88config CPU_SUBTYPE_SH7710
89 bool "Support SH7710 processor"
90 select CPU_SH3
91 select CPU_HAS_DSP
92 help
93 Select SH7710 if you have a SH3-DSP SH7710 CPU.
94
95config CPU_SUBTYPE_SH7712
96 bool "Support SH7712 processor"
97 select CPU_SH3
98 select CPU_HAS_DSP
99 help
100 Select SH7712 if you have a SH3-DSP SH7712 CPU.
101
102config CPU_SUBTYPE_SH7720
103 bool "Support SH7720 processor"
104 select CPU_SH3
105 select CPU_HAS_DSP
106 help
107 Select SH7720 if you have a SH3-DSP SH7720 CPU.
108
109# SH-4 Processor Support
110
111config CPU_SUBTYPE_SH7750
112 bool "Support SH7750 processor"
113 select CPU_SH4
114 help
115 Select SH7750 if you have a 200 Mhz SH-4 HD6417750 CPU.
116
117config CPU_SUBTYPE_SH7091
118 bool "Support SH7091 processor"
119 select CPU_SH4
120 help
121 Select SH7091 if you have an SH-4 based Sega device (such as
122 the Dreamcast, Naomi, and Naomi 2).
123
124config CPU_SUBTYPE_SH7750R
125 bool "Support SH7750R processor"
126 select CPU_SH4
127
128config CPU_SUBTYPE_SH7750S
129 bool "Support SH7750S processor"
130 select CPU_SH4
131
132config CPU_SUBTYPE_SH7751
133 bool "Support SH7751 processor"
134 select CPU_SH4
135 help
136 Select SH7751 if you have a 166 Mhz SH-4 HD6417751 CPU,
137 or if you have a HD6417751R CPU.
138
139config CPU_SUBTYPE_SH7751R
140 bool "Support SH7751R processor"
141 select CPU_SH4
142
143config CPU_SUBTYPE_SH7760
144 bool "Support SH7760 processor"
145 select CPU_SH4
146
147config CPU_SUBTYPE_SH4_202
148 bool "Support SH4-202 processor"
149 select CPU_SH4
150
151# SH-4A Processor Support
152
153config CPU_SUBTYPE_SH7770
154 bool "Support SH7770 processor"
155 select CPU_SH4A
156
157config CPU_SUBTYPE_SH7780
158 bool "Support SH7780 processor"
159 select CPU_SH4A
160
161config CPU_SUBTYPE_SH7785
162 bool "Support SH7785 processor"
163 select CPU_SH4A
164 select CPU_SHX2
165 select ARCH_SPARSEMEM_ENABLE
166 select SYS_SUPPORTS_NUMA
167
168config CPU_SUBTYPE_SHX3
169 bool "Support SH-X3 processor"
170 select CPU_SH4A
171 select CPU_SHX3
172 select ARCH_SPARSEMEM_ENABLE
173 select SYS_SUPPORTS_NUMA
174 select SYS_SUPPORTS_SMP
175
176# SH4AL-DSP Processor Support
177
178config CPU_SUBTYPE_SH7343
179 bool "Support SH7343 processor"
180 select CPU_SH4AL_DSP
181
182config CPU_SUBTYPE_SH7722
183 bool "Support SH7722 processor"
184 select CPU_SH4AL_DSP
185 select CPU_SHX2
186 select ARCH_SPARSEMEM_ENABLE
187 select SYS_SUPPORTS_NUMA
188
189endchoice
190
191menu "Memory management options" 1menu "Memory management options"
192 2
193config QUICKLIST 3config QUICKLIST
@@ -207,7 +17,8 @@ config MMU
207 17
208config PAGE_OFFSET 18config PAGE_OFFSET
209 hex 19 hex
210 default "0x80000000" if MMU 20 default "0x80000000" if MMU && SUPERH32
21 default "0x20000000" if MMU && SUPERH64
211 default "0x00000000" 22 default "0x00000000"
212 23
213config MEMORY_START 24config MEMORY_START
@@ -228,17 +39,28 @@ config MEMORY_START
228 39
229config MEMORY_SIZE 40config MEMORY_SIZE
230 hex "Physical memory size" 41 hex "Physical memory size"
231 default "0x00400000" 42 default "0x04000000"
232 help 43 help
233 This sets the default memory size assumed by your SH kernel. It can 44 This sets the default memory size assumed by your SH kernel. It can
234 be overridden as normal by the 'mem=' argument on the kernel command 45 be overridden as normal by the 'mem=' argument on the kernel command
235 line. If unsure, consult your board specifications or just leave it 46 line. If unsure, consult your board specifications or just leave it
236 as 0x00400000 which was the default value before this became 47 as 0x04000000 which was the default value before this became
237 configurable. 48 configurable.
238 49
50# Physical addressing modes
51
52config 29BIT
53 def_bool !32BIT
54 depends on SUPERH32
55
239config 32BIT 56config 32BIT
57 bool
58 default y if CPU_SH5
59
60config PMB
240 bool "Support 32-bit physical addressing through PMB" 61 bool "Support 32-bit physical addressing through PMB"
241 depends on MMU && (CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785) 62 depends on MMU && (CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785)
63 select 32BIT
242 default y 64 default y
243 help 65 help
244 If you say Y here, physical addressing will be extended to 66 If you say Y here, physical addressing will be extended to
@@ -256,7 +78,7 @@ config X2TLB
256 78
257config VSYSCALL 79config VSYSCALL
258 bool "Support vsyscall page" 80 bool "Support vsyscall page"
259 depends on MMU 81 depends on MMU && (CPU_SH3 || CPU_SH4)
260 default y 82 default y
261 help 83 help
262 This will enable support for the kernel mapping a vDSO page 84 This will enable support for the kernel mapping a vDSO page
@@ -335,7 +157,7 @@ config PAGE_SIZE_8KB
335 157
336config PAGE_SIZE_64KB 158config PAGE_SIZE_64KB
337 bool "64kB" 159 bool "64kB"
338 depends on CPU_SH4 160 depends on CPU_SH4 || CPU_SH5
339 help 161 help
340 This enables support for 64kB pages, possible on all SH-4 162 This enables support for 64kB pages, possible on all SH-4
341 CPUs and later. 163 CPUs and later.
@@ -344,7 +166,7 @@ endchoice
344 166
345choice 167choice
346 prompt "HugeTLB page size" 168 prompt "HugeTLB page size"
347 depends on HUGETLB_PAGE && CPU_SH4 && MMU 169 depends on HUGETLB_PAGE && (CPU_SH4 || CPU_SH5) && MMU
348 default HUGETLB_PAGE_SIZE_64K 170 default HUGETLB_PAGE_SIZE_64K
349 171
350config HUGETLB_PAGE_SIZE_64K 172config HUGETLB_PAGE_SIZE_64K
@@ -365,6 +187,10 @@ config HUGETLB_PAGE_SIZE_64MB
365 bool "64MB" 187 bool "64MB"
366 depends on X2TLB 188 depends on X2TLB
367 189
190config HUGETLB_PAGE_SIZE_512MB
191 bool "512MB"
192 depends on CPU_SH5
193
368endchoice 194endchoice
369 195
370source "mm/Kconfig" 196source "mm/Kconfig"
@@ -392,12 +218,12 @@ config SH_DIRECT_MAPPED
392 218
393choice 219choice
394 prompt "Cache mode" 220 prompt "Cache mode"
395 default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4 221 default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5
396 default CACHE_WRITETHROUGH if (CPU_SH2 && !CPU_SH2A) 222 default CACHE_WRITETHROUGH if (CPU_SH2 && !CPU_SH2A)
397 223
398config CACHE_WRITEBACK 224config CACHE_WRITEBACK
399 bool "Write-back" 225 bool "Write-back"
400 depends on CPU_SH2A || CPU_SH3 || CPU_SH4 226 depends on CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5
401 227
402config CACHE_WRITETHROUGH 228config CACHE_WRITETHROUGH
403 bool "Write-through" 229 bool "Write-through"
diff --git a/arch/sh/mm/Makefile b/arch/sh/mm/Makefile
index aa44607f072d..9f4bc3d90b1e 100644
--- a/arch/sh/mm/Makefile
+++ b/arch/sh/mm/Makefile
@@ -1,37 +1,5 @@
1# 1ifeq ($(CONFIG_SUPERH32),y)
2# Makefile for the Linux SuperH-specific parts of the memory manager. 2include ${srctree}/arch/sh/mm/Makefile_32
3# 3else
4 4include ${srctree}/arch/sh/mm/Makefile_64
5obj-y := init.o extable.o consistent.o
6
7ifndef CONFIG_CACHE_OFF
8obj-$(CONFIG_CPU_SH2) += cache-sh2.o
9obj-$(CONFIG_CPU_SH3) += cache-sh3.o
10obj-$(CONFIG_CPU_SH4) += cache-sh4.o
11obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o
12endif 5endif
13
14mmu-y := tlb-nommu.o pg-nommu.o
15mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush.o \
16 ioremap.o
17
18obj-y += $(mmu-y)
19
20ifdef CONFIG_DEBUG_FS
21obj-$(CONFIG_CPU_SH4) += cache-debugfs.o
22endif
23
24ifdef CONFIG_MMU
25obj-$(CONFIG_CPU_SH3) += tlb-sh3.o
26obj-$(CONFIG_CPU_SH4) += tlb-sh4.o
27ifndef CONFIG_CACHE_OFF
28obj-$(CONFIG_CPU_SH4) += pg-sh4.o
29obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o
30endif
31endif
32
33obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
34obj-$(CONFIG_32BIT) += pmb.o
35obj-$(CONFIG_NUMA) += numa.o
36
37EXTRA_CFLAGS += -Werror
diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32
new file mode 100644
index 000000000000..e295db60b91b
--- /dev/null
+++ b/arch/sh/mm/Makefile_32
@@ -0,0 +1,36 @@
1#
2# Makefile for the Linux SuperH-specific parts of the memory manager.
3#
4
5obj-y := init.o extable_32.o consistent.o
6
7ifndef CONFIG_CACHE_OFF
8obj-$(CONFIG_CPU_SH2) += cache-sh2.o
9obj-$(CONFIG_CPU_SH3) += cache-sh3.o
10obj-$(CONFIG_CPU_SH4) += cache-sh4.o
11obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o
12endif
13
14mmu-y := tlb-nommu.o pg-nommu.o
15mmu-$(CONFIG_MMU) := fault_32.o tlbflush_32.o ioremap_32.o
16
17obj-y += $(mmu-y)
18
19ifdef CONFIG_DEBUG_FS
20obj-$(CONFIG_CPU_SH4) += cache-debugfs.o
21endif
22
23ifdef CONFIG_MMU
24obj-$(CONFIG_CPU_SH3) += tlb-sh3.o
25obj-$(CONFIG_CPU_SH4) += tlb-sh4.o
26ifndef CONFIG_CACHE_OFF
27obj-$(CONFIG_CPU_SH4) += pg-sh4.o
28obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o
29endif
30endif
31
32obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
33obj-$(CONFIG_PMB) += pmb.o
34obj-$(CONFIG_NUMA) += numa.o
35
36EXTRA_CFLAGS += -Werror
diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64
new file mode 100644
index 000000000000..cbd6aa33c5ac
--- /dev/null
+++ b/arch/sh/mm/Makefile_64
@@ -0,0 +1,44 @@
1#
2# Makefile for the Linux SuperH-specific parts of the memory manager.
3#
4
5obj-y := init.o extable_64.o consistent.o
6
7mmu-y := tlb-nommu.o pg-nommu.o
8mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlbflush_64.o tlb-sh5.o
9
10ifndef CONFIG_CACHE_OFF
11obj-y += cache-sh5.o
12endif
13
14obj-y += $(mmu-y)
15
16obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
17obj-$(CONFIG_NUMA) += numa.o
18
19EXTRA_CFLAGS += -Werror
20
21# Special flags for fault_64.o. This puts restrictions on the number of
22# caller-save registers that the compiler can target when building this file.
23# This is required because the code is called from a context in entry.S where
24# very few registers have been saved in the exception handler (for speed
25# reasons).
26# The caller save registers that have been saved and which can be used are
27# r2,r3,r4,r5 : argument passing
28# r15, r18 : SP and LINK
29# tr0-4 : allow all caller-save TR's. The compiler seems to be able to make
30# use of them, so it's probably beneficial to performance to save them
31# and have them available for it.
32#
33# The resources not listed below are callee save, i.e. the compiler is free to
34# use any of them and will spill them to the stack itself.
35
36CFLAGS_fault_64.o += -ffixed-r7 \
37 -ffixed-r8 -ffixed-r9 -ffixed-r10 -ffixed-r11 -ffixed-r12 \
38 -ffixed-r13 -ffixed-r14 -ffixed-r16 -ffixed-r17 -ffixed-r19 \
39 -ffixed-r20 -ffixed-r21 -ffixed-r22 -ffixed-r23 \
40 -ffixed-r24 -ffixed-r25 -ffixed-r26 -ffixed-r27 \
41 -ffixed-r36 -ffixed-r37 -ffixed-r38 -ffixed-r39 -ffixed-r40 \
42 -ffixed-r41 -ffixed-r42 -ffixed-r43 \
43 -ffixed-r60 -ffixed-r61 -ffixed-r62 \
44 -fomit-frame-pointer
diff --git a/arch/sh/mm/cache-debugfs.c b/arch/sh/mm/cache-debugfs.c
index de6d2c9aa477..db6d950b6f5e 100644
--- a/arch/sh/mm/cache-debugfs.c
+++ b/arch/sh/mm/cache-debugfs.c
@@ -22,7 +22,8 @@ enum cache_type {
22 CACHE_TYPE_UNIFIED, 22 CACHE_TYPE_UNIFIED,
23}; 23};
24 24
25static int cache_seq_show(struct seq_file *file, void *iter) 25static int __uses_jump_to_uncached cache_seq_show(struct seq_file *file,
26 void *iter)
26{ 27{
27 unsigned int cache_type = (unsigned int)file->private; 28 unsigned int cache_type = (unsigned int)file->private;
28 struct cache_info *cache; 29 struct cache_info *cache;
@@ -34,11 +35,11 @@ static int cache_seq_show(struct seq_file *file, void *iter)
34 * Go uncached immediately so we don't skew the results any 35 * Go uncached immediately so we don't skew the results any
35 * more than we already are.. 36 * more than we already are..
36 */ 37 */
37 jump_to_P2(); 38 jump_to_uncached();
38 39
39 ccr = ctrl_inl(CCR); 40 ccr = ctrl_inl(CCR);
40 if ((ccr & CCR_CACHE_ENABLE) == 0) { 41 if ((ccr & CCR_CACHE_ENABLE) == 0) {
41 back_to_P1(); 42 back_to_cached();
42 43
43 seq_printf(file, "disabled\n"); 44 seq_printf(file, "disabled\n");
44 return 0; 45 return 0;
@@ -104,7 +105,7 @@ static int cache_seq_show(struct seq_file *file, void *iter)
104 addrstart += cache->way_incr; 105 addrstart += cache->way_incr;
105 } 106 }
106 107
107 back_to_P1(); 108 back_to_cached();
108 109
109 return 0; 110 return 0;
110} 111}
diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c
index 226b190c5b9c..43d7ff6b6ec7 100644
--- a/arch/sh/mm/cache-sh4.c
+++ b/arch/sh/mm/cache-sh4.c
@@ -190,7 +190,7 @@ void flush_icache_range(unsigned long start, unsigned long end)
190 * .. which happens to be the same behavior as flush_icache_range(). 190 * .. which happens to be the same behavior as flush_icache_range().
191 * So, we simply flush out a line. 191 * So, we simply flush out a line.
192 */ 192 */
193void flush_cache_sigtramp(unsigned long addr) 193void __uses_jump_to_uncached flush_cache_sigtramp(unsigned long addr)
194{ 194{
195 unsigned long v, index; 195 unsigned long v, index;
196 unsigned long flags; 196 unsigned long flags;
@@ -205,13 +205,13 @@ void flush_cache_sigtramp(unsigned long addr)
205 (v & boot_cpu_data.icache.entry_mask); 205 (v & boot_cpu_data.icache.entry_mask);
206 206
207 local_irq_save(flags); 207 local_irq_save(flags);
208 jump_to_P2(); 208 jump_to_uncached();
209 209
210 for (i = 0; i < boot_cpu_data.icache.ways; 210 for (i = 0; i < boot_cpu_data.icache.ways;
211 i++, index += boot_cpu_data.icache.way_incr) 211 i++, index += boot_cpu_data.icache.way_incr)
212 ctrl_outl(0, index); /* Clear out Valid-bit */ 212 ctrl_outl(0, index); /* Clear out Valid-bit */
213 213
214 back_to_P1(); 214 back_to_cached();
215 wmb(); 215 wmb();
216 local_irq_restore(flags); 216 local_irq_restore(flags);
217} 217}
@@ -256,12 +256,12 @@ void flush_dcache_page(struct page *page)
256} 256}
257 257
258/* TODO: Selective icache invalidation through IC address array.. */ 258/* TODO: Selective icache invalidation through IC address array.. */
259static inline void flush_icache_all(void) 259static inline void __uses_jump_to_uncached flush_icache_all(void)
260{ 260{
261 unsigned long flags, ccr; 261 unsigned long flags, ccr;
262 262
263 local_irq_save(flags); 263 local_irq_save(flags);
264 jump_to_P2(); 264 jump_to_uncached();
265 265
266 /* Flush I-cache */ 266 /* Flush I-cache */
267 ccr = ctrl_inl(CCR); 267 ccr = ctrl_inl(CCR);
@@ -269,11 +269,11 @@ static inline void flush_icache_all(void)
269 ctrl_outl(ccr, CCR); 269 ctrl_outl(ccr, CCR);
270 270
271 /* 271 /*
272 * back_to_P1() will take care of the barrier for us, don't add 272 * back_to_cached() will take care of the barrier for us, don't add
273 * another one! 273 * another one!
274 */ 274 */
275 275
276 back_to_P1(); 276 back_to_cached();
277 local_irq_restore(flags); 277 local_irq_restore(flags);
278} 278}
279 279
diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c
new file mode 100644
index 000000000000..4617e3aeee73
--- /dev/null
+++ b/arch/sh/mm/cache-sh5.c
@@ -0,0 +1,1029 @@
1/*
2 * arch/sh/mm/cache-sh5.c
3 *
4 * Original version Copyright (C) 2000, 2001 Paolo Alberelli
5 * Second version Copyright (C) benedict.gaster@superh.com 2002
6 * Third version Copyright Richard.Curnow@superh.com 2003
7 * Hacks to third version Copyright (C) 2003 Paul Mundt
8 *
9 * This file is subject to the terms and conditions of the GNU General Public
10 * License. See the file "COPYING" in the main directory of this archive
11 * for more details.
12 */
13#include <linux/init.h>
14#include <linux/mman.h>
15#include <linux/mm.h>
16#include <linux/threads.h>
17#include <asm/page.h>
18#include <asm/pgtable.h>
19#include <asm/processor.h>
20#include <asm/cache.h>
21#include <asm/tlb.h>
22#include <asm/io.h>
23#include <asm/uaccess.h>
24#include <asm/mmu_context.h>
25#include <asm/pgalloc.h> /* for flush_itlb_range */
26
27#include <linux/proc_fs.h>
28
29/* This function is in entry.S */
30extern unsigned long switch_and_save_asid(unsigned long new_asid);
31
32/* Wired TLB entry for the D-cache */
33static unsigned long long dtlb_cache_slot;
34
35/**
36 * sh64_cache_init()
37 *
38 * This is pretty much just a straightforward clone of the SH
39 * detect_cpu_and_cache_system().
40 *
41 * This function is responsible for setting up all of the cache
42 * info dynamically as well as taking care of CPU probing and
43 * setting up the relevant subtype data.
44 *
45 * FIXME: For the time being, we only really support the SH5-101
46 * out of the box, and don't support dynamic probing for things
47 * like the SH5-103 or even cut2 of the SH5-101. Implement this
48 * later!
49 */
50int __init sh64_cache_init(void)
51{
52 /*
53 * First, setup some sane values for the I-cache.
54 */
55 cpu_data->icache.ways = 4;
56 cpu_data->icache.sets = 256;
57 cpu_data->icache.linesz = L1_CACHE_BYTES;
58
59 /*
60 * FIXME: This can probably be cleaned up a bit as well.. for example,
61 * do we really need the way shift _and_ the way_step_shift ?? Judging
62 * by the existing code, I would guess no.. is there any valid reason
63 * why we need to be tracking this around?
64 */
65 cpu_data->icache.way_shift = 13;
66 cpu_data->icache.entry_shift = 5;
67 cpu_data->icache.set_shift = 4;
68 cpu_data->icache.way_step_shift = 16;
69 cpu_data->icache.asid_shift = 2;
70
71 /*
72 * way offset = cache size / associativity, so just don't factor in
73 * associativity in the first place..
74 */
75 cpu_data->icache.way_ofs = cpu_data->icache.sets *
76 cpu_data->icache.linesz;
77
78 cpu_data->icache.asid_mask = 0x3fc;
79 cpu_data->icache.idx_mask = 0x1fe0;
80 cpu_data->icache.epn_mask = 0xffffe000;
81 cpu_data->icache.flags = 0;
82
83 /*
84 * Next, setup some sane values for the D-cache.
85 *
86 * On the SH5, these are pretty consistent with the I-cache settings,
87 * so we just copy over the existing definitions.. these can be fixed
88 * up later, especially if we add runtime CPU probing.
89 *
90 * Though in the meantime it saves us from having to duplicate all of
91 * the above definitions..
92 */
93 cpu_data->dcache = cpu_data->icache;
94
95 /*
96 * Setup any cache-related flags here
97 */
98#if defined(CONFIG_DCACHE_WRITE_THROUGH)
99 set_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags));
100#elif defined(CONFIG_DCACHE_WRITE_BACK)
101 set_bit(SH_CACHE_MODE_WB, &(cpu_data->dcache.flags));
102#endif
103
104 /*
105 * We also need to reserve a slot for the D-cache in the DTLB, so we
106 * do this now ..
107 */
108 dtlb_cache_slot = sh64_get_wired_dtlb_entry();
109
110 return 0;
111}
112
113#ifdef CONFIG_DCACHE_DISABLED
114#define sh64_dcache_purge_all() do { } while (0)
115#define sh64_dcache_purge_coloured_phy_page(paddr, eaddr) do { } while (0)
116#define sh64_dcache_purge_user_range(mm, start, end) do { } while (0)
117#define sh64_dcache_purge_phy_page(paddr) do { } while (0)
118#define sh64_dcache_purge_virt_page(mm, eaddr) do { } while (0)
119#define sh64_dcache_purge_kernel_range(start, end) do { } while (0)
120#define sh64_dcache_wback_current_user_range(start, end) do { } while (0)
121#endif
122
123/*##########################################################################*/
124
125/* From here onwards, a rewrite of the implementation,
126 by Richard.Curnow@superh.com.
127
128 The major changes in this compared to the old version are;
129 1. use more selective purging through OCBP instead of using ALLOCO to purge
130 by natural replacement. This avoids purging out unrelated cache lines
131 that happen to be in the same set.
132 2. exploit the APIs copy_user_page and clear_user_page better
133 3. be more selective about I-cache purging, in particular use invalidate_all
134 more sparingly.
135
136 */
137
138/*##########################################################################
139 SUPPORT FUNCTIONS
140 ##########################################################################*/
141
142/****************************************************************************/
143/* The following group of functions deal with mapping and unmapping a temporary
144 page into the DTLB slot that have been set aside for our exclusive use. */
145/* In order to accomplish this, we use the generic interface for adding and
146 removing a wired slot entry as defined in arch/sh/mm/tlb-sh5.c */
147/****************************************************************************/
148
149static unsigned long slot_own_flags;
150
151static inline void sh64_setup_dtlb_cache_slot(unsigned long eaddr, unsigned long asid, unsigned long paddr)
152{
153 local_irq_save(slot_own_flags);
154 sh64_setup_tlb_slot(dtlb_cache_slot, eaddr, asid, paddr);
155}
156
157static inline void sh64_teardown_dtlb_cache_slot(void)
158{
159 sh64_teardown_tlb_slot(dtlb_cache_slot);
160 local_irq_restore(slot_own_flags);
161}
162
163/****************************************************************************/
164
165#ifndef CONFIG_ICACHE_DISABLED
166
167static void __inline__ sh64_icache_inv_all(void)
168{
169 unsigned long long addr, flag, data;
170 unsigned int flags;
171
172 addr=ICCR0;
173 flag=ICCR0_ICI;
174 data=0;
175
176 /* Make this a critical section for safety (probably not strictly necessary.) */
177 local_irq_save(flags);
178
179 /* Without %1 it gets unexplicably wrong */
180 asm volatile("getcfg %3, 0, %0\n\t"
181 "or %0, %2, %0\n\t"
182 "putcfg %3, 0, %0\n\t"
183 "synci"
184 : "=&r" (data)
185 : "0" (data), "r" (flag), "r" (addr));
186
187 local_irq_restore(flags);
188}
189
190static void sh64_icache_inv_kernel_range(unsigned long start, unsigned long end)
191{
192 /* Invalidate range of addresses [start,end] from the I-cache, where
193 * the addresses lie in the kernel superpage. */
194
195 unsigned long long ullend, addr, aligned_start;
196#if (NEFF == 32)
197 aligned_start = (unsigned long long)(signed long long)(signed long) start;
198#else
199#error "NEFF != 32"
200#endif
201 aligned_start &= L1_CACHE_ALIGN_MASK;
202 addr = aligned_start;
203#if (NEFF == 32)
204 ullend = (unsigned long long) (signed long long) (signed long) end;
205#else
206#error "NEFF != 32"
207#endif
208 while (addr <= ullend) {
209 asm __volatile__ ("icbi %0, 0" : : "r" (addr));
210 addr += L1_CACHE_BYTES;
211 }
212}
213
214static void sh64_icache_inv_user_page(struct vm_area_struct *vma, unsigned long eaddr)
215{
216 /* If we get called, we know that vma->vm_flags contains VM_EXEC.
217 Also, eaddr is page-aligned. */
218
219 unsigned long long addr, end_addr;
220 unsigned long flags = 0;
221 unsigned long running_asid, vma_asid;
222 addr = eaddr;
223 end_addr = addr + PAGE_SIZE;
224
225 /* Check whether we can use the current ASID for the I-cache
226 invalidation. For example, if we're called via
227 access_process_vm->flush_cache_page->here, (e.g. when reading from
228 /proc), 'running_asid' will be that of the reader, not of the
229 victim.
230
231 Also, note the risk that we might get pre-empted between the ASID
232 compare and blocking IRQs, and before we regain control, the
233 pid->ASID mapping changes. However, the whole cache will get
234 invalidated when the mapping is renewed, so the worst that can
235 happen is that the loop below ends up invalidating somebody else's
236 cache entries.
237 */
238
239 running_asid = get_asid();
240 vma_asid = (vma->vm_mm->context & MMU_CONTEXT_ASID_MASK);
241 if (running_asid != vma_asid) {
242 local_irq_save(flags);
243 switch_and_save_asid(vma_asid);
244 }
245 while (addr < end_addr) {
246 /* Worth unrolling a little */
247 asm __volatile__("icbi %0, 0" : : "r" (addr));
248 asm __volatile__("icbi %0, 32" : : "r" (addr));
249 asm __volatile__("icbi %0, 64" : : "r" (addr));
250 asm __volatile__("icbi %0, 96" : : "r" (addr));
251 addr += 128;
252 }
253 if (running_asid != vma_asid) {
254 switch_and_save_asid(running_asid);
255 local_irq_restore(flags);
256 }
257}
258
259/****************************************************************************/
260
261static void sh64_icache_inv_user_page_range(struct mm_struct *mm,
262 unsigned long start, unsigned long end)
263{
264 /* Used for invalidating big chunks of I-cache, i.e. assume the range
265 is whole pages. If 'start' or 'end' is not page aligned, the code
266 is conservative and invalidates to the ends of the enclosing pages.
267 This is functionally OK, just a performance loss. */
268
269 /* See the comments below in sh64_dcache_purge_user_range() regarding
270 the choice of algorithm. However, for the I-cache option (2) isn't
271 available because there are no physical tags so aliases can't be
272 resolved. The icbi instruction has to be used through the user
273 mapping. Because icbi is cheaper than ocbp on a cache hit, it
274 would be cheaper to use the selective code for a large range than is
275 possible with the D-cache. Just assume 64 for now as a working
276 figure.
277 */
278
279 int n_pages;
280
281 if (!mm) return;
282
283 n_pages = ((end - start) >> PAGE_SHIFT);
284 if (n_pages >= 64) {
285 sh64_icache_inv_all();
286 } else {
287 unsigned long aligned_start;
288 unsigned long eaddr;
289 unsigned long after_last_page_start;
290 unsigned long mm_asid, current_asid;
291 unsigned long long flags = 0ULL;
292
293 mm_asid = mm->context & MMU_CONTEXT_ASID_MASK;
294 current_asid = get_asid();
295
296 if (mm_asid != current_asid) {
297 /* Switch ASID and run the invalidate loop under cli */
298 local_irq_save(flags);
299 switch_and_save_asid(mm_asid);
300 }
301
302 aligned_start = start & PAGE_MASK;
303 after_last_page_start = PAGE_SIZE + ((end - 1) & PAGE_MASK);
304
305 while (aligned_start < after_last_page_start) {
306 struct vm_area_struct *vma;
307 unsigned long vma_end;
308 vma = find_vma(mm, aligned_start);
309 if (!vma || (aligned_start <= vma->vm_end)) {
310 /* Avoid getting stuck in an error condition */
311 aligned_start += PAGE_SIZE;
312 continue;
313 }
314 vma_end = vma->vm_end;
315 if (vma->vm_flags & VM_EXEC) {
316 /* Executable */
317 eaddr = aligned_start;
318 while (eaddr < vma_end) {
319 sh64_icache_inv_user_page(vma, eaddr);
320 eaddr += PAGE_SIZE;
321 }
322 }
323 aligned_start = vma->vm_end; /* Skip to start of next region */
324 }
325 if (mm_asid != current_asid) {
326 switch_and_save_asid(current_asid);
327 local_irq_restore(flags);
328 }
329 }
330}
331
332static void sh64_icache_inv_user_small_range(struct mm_struct *mm,
333 unsigned long start, int len)
334{
335
336 /* Invalidate a small range of user context I-cache, not necessarily
337 page (or even cache-line) aligned. */
338
339 unsigned long long eaddr = start;
340 unsigned long long eaddr_end = start + len;
341 unsigned long current_asid, mm_asid;
342 unsigned long long flags;
343 unsigned long long epage_start;
344
345 /* Since this is used inside ptrace, the ASID in the mm context
346 typically won't match current_asid. We'll have to switch ASID to do
347 this. For safety, and given that the range will be small, do all
348 this under cli.
349
350 Note, there is a hazard that the ASID in mm->context is no longer
351 actually associated with mm, i.e. if the mm->context has started a
352 new cycle since mm was last active. However, this is just a
353 performance issue: all that happens is that we invalidate lines
354 belonging to another mm, so the owning process has to refill them
355 when that mm goes live again. mm itself can't have any cache
356 entries because there will have been a flush_cache_all when the new
357 mm->context cycle started. */
358
359 /* Align to start of cache line. Otherwise, suppose len==8 and start
360 was at 32N+28 : the last 4 bytes wouldn't get invalidated. */
361 eaddr = start & L1_CACHE_ALIGN_MASK;
362 eaddr_end = start + len;
363
364 local_irq_save(flags);
365 mm_asid = mm->context & MMU_CONTEXT_ASID_MASK;
366 current_asid = switch_and_save_asid(mm_asid);
367
368 epage_start = eaddr & PAGE_MASK;
369
370 while (eaddr < eaddr_end)
371 {
372 asm __volatile__("icbi %0, 0" : : "r" (eaddr));
373 eaddr += L1_CACHE_BYTES;
374 }
375 switch_and_save_asid(current_asid);
376 local_irq_restore(flags);
377}
378
379static void sh64_icache_inv_current_user_range(unsigned long start, unsigned long end)
380{
381 /* The icbi instruction never raises ITLBMISS. i.e. if there's not a
382 cache hit on the virtual tag the instruction ends there, without a
383 TLB lookup. */
384
385 unsigned long long aligned_start;
386 unsigned long long ull_end;
387 unsigned long long addr;
388
389 ull_end = end;
390
391 /* Just invalidate over the range using the natural addresses. TLB
392 miss handling will be OK (TBC). Since it's for the current process,
393 either we're already in the right ASID context, or the ASIDs have
394 been recycled since we were last active in which case we might just
395 invalidate another processes I-cache entries : no worries, just a
396 performance drop for him. */
397 aligned_start = start & L1_CACHE_ALIGN_MASK;
398 addr = aligned_start;
399 while (addr < ull_end) {
400 asm __volatile__ ("icbi %0, 0" : : "r" (addr));
401 asm __volatile__ ("nop");
402 asm __volatile__ ("nop");
403 addr += L1_CACHE_BYTES;
404 }
405}
406
407#endif /* !CONFIG_ICACHE_DISABLED */
408
409/****************************************************************************/
410
411#ifndef CONFIG_DCACHE_DISABLED
412
413/* Buffer used as the target of alloco instructions to purge data from cache
414 sets by natural eviction. -- RPC */
415#define DUMMY_ALLOCO_AREA_SIZE L1_CACHE_SIZE_BYTES + (1024 * 4)
416static unsigned char dummy_alloco_area[DUMMY_ALLOCO_AREA_SIZE] __cacheline_aligned = { 0, };
417
418/****************************************************************************/
419
420static void __inline__ sh64_dcache_purge_sets(int sets_to_purge_base, int n_sets)
421{
422 /* Purge all ways in a particular block of sets, specified by the base
423 set number and number of sets. Can handle wrap-around, if that's
424 needed. */
425
426 int dummy_buffer_base_set;
427 unsigned long long eaddr, eaddr0, eaddr1;
428 int j;
429 int set_offset;
430
431 dummy_buffer_base_set = ((int)&dummy_alloco_area & cpu_data->dcache.idx_mask) >> cpu_data->dcache.entry_shift;
432 set_offset = sets_to_purge_base - dummy_buffer_base_set;
433
434 for (j=0; j<n_sets; j++, set_offset++) {
435 set_offset &= (cpu_data->dcache.sets - 1);
436 eaddr0 = (unsigned long long)dummy_alloco_area + (set_offset << cpu_data->dcache.entry_shift);
437
438 /* Do one alloco which hits the required set per cache way. For
439 write-back mode, this will purge the #ways resident lines. There's
440 little point unrolling this loop because the allocos stall more if
441 they're too close together. */
442 eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways;
443 for (eaddr=eaddr0; eaddr<eaddr1; eaddr+=cpu_data->dcache.way_ofs) {
444 asm __volatile__ ("alloco %0, 0" : : "r" (eaddr));
445 asm __volatile__ ("synco"); /* TAKum03020 */
446 }
447
448 eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways;
449 for (eaddr=eaddr0; eaddr<eaddr1; eaddr+=cpu_data->dcache.way_ofs) {
450 /* Load from each address. Required because alloco is a NOP if
451 the cache is write-through. Write-through is a config option. */
452 if (test_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags)))
453 *(volatile unsigned char *)(int)eaddr;
454 }
455 }
456
457 /* Don't use OCBI to invalidate the lines. That costs cycles directly.
458 If the dummy block is just left resident, it will naturally get
459 evicted as required. */
460
461 return;
462}
463
464/****************************************************************************/
465
466static void sh64_dcache_purge_all(void)
467{
468 /* Purge the entire contents of the dcache. The most efficient way to
469 achieve this is to use alloco instructions on a region of unused
470 memory equal in size to the cache, thereby causing the current
471 contents to be discarded by natural eviction. The alternative,
472 namely reading every tag, setting up a mapping for the corresponding
473 page and doing an OCBP for the line, would be much more expensive.
474 */
475
476 sh64_dcache_purge_sets(0, cpu_data->dcache.sets);
477
478 return;
479
480}
481
482/****************************************************************************/
483
484static void sh64_dcache_purge_kernel_range(unsigned long start, unsigned long end)
485{
486 /* Purge the range of addresses [start,end] from the D-cache. The
487 addresses lie in the superpage mapping. There's no harm if we
488 overpurge at either end - just a small performance loss. */
489 unsigned long long ullend, addr, aligned_start;
490#if (NEFF == 32)
491 aligned_start = (unsigned long long)(signed long long)(signed long) start;
492#else
493#error "NEFF != 32"
494#endif
495 aligned_start &= L1_CACHE_ALIGN_MASK;
496 addr = aligned_start;
497#if (NEFF == 32)
498 ullend = (unsigned long long) (signed long long) (signed long) end;
499#else
500#error "NEFF != 32"
501#endif
502 while (addr <= ullend) {
503 asm __volatile__ ("ocbp %0, 0" : : "r" (addr));
504 addr += L1_CACHE_BYTES;
505 }
506 return;
507}
508
509/* Assumes this address (+ (2**n_synbits) pages up from it) aren't used for
510 anything else in the kernel */
511#define MAGIC_PAGE0_START 0xffffffffec000000ULL
512
513static void sh64_dcache_purge_coloured_phy_page(unsigned long paddr, unsigned long eaddr)
514{
515 /* Purge the physical page 'paddr' from the cache. It's known that any
516 cache lines requiring attention have the same page colour as the the
517 address 'eaddr'.
518
519 This relies on the fact that the D-cache matches on physical tags
520 when no virtual tag matches. So we create an alias for the original
521 page and purge through that. (Alternatively, we could have done
522 this by switching ASID to match the original mapping and purged
523 through that, but that involves ASID switching cost + probably a
524 TLBMISS + refill anyway.)
525 */
526
527 unsigned long long magic_page_start;
528 unsigned long long magic_eaddr, magic_eaddr_end;
529
530 magic_page_start = MAGIC_PAGE0_START + (eaddr & CACHE_OC_SYN_MASK);
531
532 /* As long as the kernel is not pre-emptible, this doesn't need to be
533 under cli/sti. */
534
535 sh64_setup_dtlb_cache_slot(magic_page_start, get_asid(), paddr);
536
537 magic_eaddr = magic_page_start;
538 magic_eaddr_end = magic_eaddr + PAGE_SIZE;
539 while (magic_eaddr < magic_eaddr_end) {
540 /* Little point in unrolling this loop - the OCBPs are blocking
541 and won't go any quicker (i.e. the loop overhead is parallel
542 to part of the OCBP execution.) */
543 asm __volatile__ ("ocbp %0, 0" : : "r" (magic_eaddr));
544 magic_eaddr += L1_CACHE_BYTES;
545 }
546
547 sh64_teardown_dtlb_cache_slot();
548}
549
550/****************************************************************************/
551
552static void sh64_dcache_purge_phy_page(unsigned long paddr)
553{
554 /* Pure a page given its physical start address, by creating a
555 temporary 1 page mapping and purging across that. Even if we know
556 the virtual address (& vma or mm) of the page, the method here is
557 more elegant because it avoids issues of coping with page faults on
558 the purge instructions (i.e. no special-case code required in the
559 critical path in the TLB miss handling). */
560
561 unsigned long long eaddr_start, eaddr, eaddr_end;
562 int i;
563
564 /* As long as the kernel is not pre-emptible, this doesn't need to be
565 under cli/sti. */
566
567 eaddr_start = MAGIC_PAGE0_START;
568 for (i=0; i < (1 << CACHE_OC_N_SYNBITS); i++) {
569 sh64_setup_dtlb_cache_slot(eaddr_start, get_asid(), paddr);
570
571 eaddr = eaddr_start;
572 eaddr_end = eaddr + PAGE_SIZE;
573 while (eaddr < eaddr_end) {
574 asm __volatile__ ("ocbp %0, 0" : : "r" (eaddr));
575 eaddr += L1_CACHE_BYTES;
576 }
577
578 sh64_teardown_dtlb_cache_slot();
579 eaddr_start += PAGE_SIZE;
580 }
581}
582
583static void sh64_dcache_purge_user_pages(struct mm_struct *mm,
584 unsigned long addr, unsigned long end)
585{
586 pgd_t *pgd;
587 pmd_t *pmd;
588 pte_t *pte;
589 pte_t entry;
590 spinlock_t *ptl;
591 unsigned long paddr;
592
593 if (!mm)
594 return; /* No way to find physical address of page */
595
596 pgd = pgd_offset(mm, addr);
597 if (pgd_bad(*pgd))
598 return;
599
600 pmd = pmd_offset(pgd, addr);
601 if (pmd_none(*pmd) || pmd_bad(*pmd))
602 return;
603
604 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
605 do {
606 entry = *pte;
607 if (pte_none(entry) || !pte_present(entry))
608 continue;
609 paddr = pte_val(entry) & PAGE_MASK;
610 sh64_dcache_purge_coloured_phy_page(paddr, addr);
611 } while (pte++, addr += PAGE_SIZE, addr != end);
612 pte_unmap_unlock(pte - 1, ptl);
613}
614/****************************************************************************/
615
616static void sh64_dcache_purge_user_range(struct mm_struct *mm,
617 unsigned long start, unsigned long end)
618{
619 /* There are at least 5 choices for the implementation of this, with
620 pros (+), cons(-), comments(*):
621
622 1. ocbp each line in the range through the original user's ASID
623 + no lines spuriously evicted
624 - tlbmiss handling (must either handle faults on demand => extra
625 special-case code in tlbmiss critical path), or map the page in
626 advance (=> flush_tlb_range in advance to avoid multiple hits)
627 - ASID switching
628 - expensive for large ranges
629
630 2. temporarily map each page in the range to a special effective
631 address and ocbp through the temporary mapping; relies on the
632 fact that SH-5 OCB* always do TLB lookup and match on ptags (they
633 never look at the etags)
634 + no spurious evictions
635 - expensive for large ranges
636 * surely cheaper than (1)
637
638 3. walk all the lines in the cache, check the tags, if a match
639 occurs create a page mapping to ocbp the line through
640 + no spurious evictions
641 - tag inspection overhead
642 - (especially for small ranges)
643 - potential cost of setting up/tearing down page mapping for
644 every line that matches the range
645 * cost partly independent of range size
646
647 4. walk all the lines in the cache, check the tags, if a match
648 occurs use 4 * alloco to purge the line (+3 other probably
649 innocent victims) by natural eviction
650 + no tlb mapping overheads
651 - spurious evictions
652 - tag inspection overhead
653
654 5. implement like flush_cache_all
655 + no tag inspection overhead
656 - spurious evictions
657 - bad for small ranges
658
659 (1) can be ruled out as more expensive than (2). (2) appears best
660 for small ranges. The choice between (3), (4) and (5) for large
661 ranges and the range size for the large/small boundary need
662 benchmarking to determine.
663
664 For now use approach (2) for small ranges and (5) for large ones.
665
666 */
667
668 int n_pages;
669
670 n_pages = ((end - start) >> PAGE_SHIFT);
671 if (n_pages >= 64 || ((start ^ (end - 1)) & PMD_MASK)) {
672#if 1
673 sh64_dcache_purge_all();
674#else
675 unsigned long long set, way;
676 unsigned long mm_asid = mm->context & MMU_CONTEXT_ASID_MASK;
677 for (set = 0; set < cpu_data->dcache.sets; set++) {
678 unsigned long long set_base_config_addr = CACHE_OC_ADDRESS_ARRAY + (set << cpu_data->dcache.set_shift);
679 for (way = 0; way < cpu_data->dcache.ways; way++) {
680 unsigned long long config_addr = set_base_config_addr + (way << cpu_data->dcache.way_step_shift);
681 unsigned long long tag0;
682 unsigned long line_valid;
683
684 asm __volatile__("getcfg %1, 0, %0" : "=r" (tag0) : "r" (config_addr));
685 line_valid = tag0 & SH_CACHE_VALID;
686 if (line_valid) {
687 unsigned long cache_asid;
688 unsigned long epn;
689
690 cache_asid = (tag0 & cpu_data->dcache.asid_mask) >> cpu_data->dcache.asid_shift;
691 /* The next line needs some
692 explanation. The virtual tags
693 encode bits [31:13] of the virtual
694 address, bit [12] of the 'tag' being
695 implied by the cache set index. */
696 epn = (tag0 & cpu_data->dcache.epn_mask) | ((set & 0x80) << cpu_data->dcache.entry_shift);
697
698 if ((cache_asid == mm_asid) && (start <= epn) && (epn < end)) {
699 /* TODO : could optimise this
700 call by batching multiple
701 adjacent sets together. */
702 sh64_dcache_purge_sets(set, 1);
703 break; /* Don't waste time inspecting other ways for this set */
704 }
705 }
706 }
707 }
708#endif
709 } else {
710 /* Small range, covered by a single page table page */
711 start &= PAGE_MASK; /* should already be so */
712 end = PAGE_ALIGN(end); /* should already be so */
713 sh64_dcache_purge_user_pages(mm, start, end);
714 }
715 return;
716}
717
718static void sh64_dcache_wback_current_user_range(unsigned long start, unsigned long end)
719{
720 unsigned long long aligned_start;
721 unsigned long long ull_end;
722 unsigned long long addr;
723
724 ull_end = end;
725
726 /* Just wback over the range using the natural addresses. TLB miss
727 handling will be OK (TBC) : the range has just been written to by
728 the signal frame setup code, so the PTEs must exist.
729
730 Note, if we have CONFIG_PREEMPT and get preempted inside this loop,
731 it doesn't matter, even if the pid->ASID mapping changes whilst
732 we're away. In that case the cache will have been flushed when the
733 mapping was renewed. So the writebacks below will be nugatory (and
734 we'll doubtless have to fault the TLB entry/ies in again with the
735 new ASID), but it's a rare case.
736 */
737 aligned_start = start & L1_CACHE_ALIGN_MASK;
738 addr = aligned_start;
739 while (addr < ull_end) {
740 asm __volatile__ ("ocbwb %0, 0" : : "r" (addr));
741 addr += L1_CACHE_BYTES;
742 }
743}
744
745/****************************************************************************/
746
747/* These *MUST* lie in an area of virtual address space that's otherwise unused. */
748#define UNIQUE_EADDR_START 0xe0000000UL
749#define UNIQUE_EADDR_END 0xe8000000UL
750
751static unsigned long sh64_make_unique_eaddr(unsigned long user_eaddr, unsigned long paddr)
752{
753 /* Given a physical address paddr, and a user virtual address
754 user_eaddr which will eventually be mapped to it, create a one-off
755 kernel-private eaddr mapped to the same paddr. This is used for
756 creating special destination pages for copy_user_page and
757 clear_user_page */
758
759 static unsigned long current_pointer = UNIQUE_EADDR_START;
760 unsigned long coloured_pointer;
761
762 if (current_pointer == UNIQUE_EADDR_END) {
763 sh64_dcache_purge_all();
764 current_pointer = UNIQUE_EADDR_START;
765 }
766
767 coloured_pointer = (current_pointer & ~CACHE_OC_SYN_MASK) | (user_eaddr & CACHE_OC_SYN_MASK);
768 sh64_setup_dtlb_cache_slot(coloured_pointer, get_asid(), paddr);
769
770 current_pointer += (PAGE_SIZE << CACHE_OC_N_SYNBITS);
771
772 return coloured_pointer;
773}
774
775/****************************************************************************/
776
777static void sh64_copy_user_page_coloured(void *to, void *from, unsigned long address)
778{
779 void *coloured_to;
780
781 /* Discard any existing cache entries of the wrong colour. These are
782 present quite often, if the kernel has recently used the page
783 internally, then given it up, then it's been allocated to the user.
784 */
785 sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to);
786
787 coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to));
788 sh64_page_copy(from, coloured_to);
789
790 sh64_teardown_dtlb_cache_slot();
791}
792
793static void sh64_clear_user_page_coloured(void *to, unsigned long address)
794{
795 void *coloured_to;
796
797 /* Discard any existing kernel-originated lines of the wrong colour (as
798 above) */
799 sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to);
800
801 coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to));
802 sh64_page_clear(coloured_to);
803
804 sh64_teardown_dtlb_cache_slot();
805}
806
807#endif /* !CONFIG_DCACHE_DISABLED */
808
809/****************************************************************************/
810
811/*##########################################################################
812 EXTERNALLY CALLABLE API.
813 ##########################################################################*/
814
815/* These functions are described in Documentation/cachetlb.txt.
816 Each one of these functions varies in behaviour depending on whether the
817 I-cache and/or D-cache are configured out.
818
819 Note that the Linux term 'flush' corresponds to what is termed 'purge' in
820 the sh/sh64 jargon for the D-cache, i.e. write back dirty data then
821 invalidate the cache lines, and 'invalidate' for the I-cache.
822 */
823
824#undef FLUSH_TRACE
825
826void flush_cache_all(void)
827{
828 /* Invalidate the entire contents of both caches, after writing back to
829 memory any dirty data from the D-cache. */
830 sh64_dcache_purge_all();
831 sh64_icache_inv_all();
832}
833
834/****************************************************************************/
835
836void flush_cache_mm(struct mm_struct *mm)
837{
838 /* Invalidate an entire user-address space from both caches, after
839 writing back dirty data (e.g. for shared mmap etc). */
840
841 /* This could be coded selectively by inspecting all the tags then
842 doing 4*alloco on any set containing a match (as for
843 flush_cache_range), but fork/exit/execve (where this is called from)
844 are expensive anyway. */
845
846 /* Have to do a purge here, despite the comments re I-cache below.
847 There could be odd-coloured dirty data associated with the mm still
848 in the cache - if this gets written out through natural eviction
849 after the kernel has reused the page there will be chaos.
850 */
851
852 sh64_dcache_purge_all();
853
854 /* The mm being torn down won't ever be active again, so any Icache
855 lines tagged with its ASID won't be visible for the rest of the
856 lifetime of this ASID cycle. Before the ASID gets reused, there
857 will be a flush_cache_all. Hence we don't need to touch the
858 I-cache. This is similar to the lack of action needed in
859 flush_tlb_mm - see fault.c. */
860}
861
862/****************************************************************************/
863
864void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
865 unsigned long end)
866{
867 struct mm_struct *mm = vma->vm_mm;
868
869 /* Invalidate (from both caches) the range [start,end) of virtual
870 addresses from the user address space specified by mm, after writing
871 back any dirty data.
872
873 Note, 'end' is 1 byte beyond the end of the range to flush. */
874
875 sh64_dcache_purge_user_range(mm, start, end);
876 sh64_icache_inv_user_page_range(mm, start, end);
877}
878
879/****************************************************************************/
880
881void flush_cache_page(struct vm_area_struct *vma, unsigned long eaddr, unsigned long pfn)
882{
883 /* Invalidate any entries in either cache for the vma within the user
884 address space vma->vm_mm for the page starting at virtual address
885 'eaddr'. This seems to be used primarily in breaking COW. Note,
886 the I-cache must be searched too in case the page in question is
887 both writable and being executed from (e.g. stack trampolines.)
888
889 Note, this is called with pte lock held.
890 */
891
892 sh64_dcache_purge_phy_page(pfn << PAGE_SHIFT);
893
894 if (vma->vm_flags & VM_EXEC) {
895 sh64_icache_inv_user_page(vma, eaddr);
896 }
897}
898
899/****************************************************************************/
900
901#ifndef CONFIG_DCACHE_DISABLED
902
903void copy_user_page(void *to, void *from, unsigned long address, struct page *page)
904{
905 /* 'from' and 'to' are kernel virtual addresses (within the superpage
906 mapping of the physical RAM). 'address' is the user virtual address
907 where the copy 'to' will be mapped after. This allows a custom
908 mapping to be used to ensure that the new copy is placed in the
909 right cache sets for the user to see it without having to bounce it
910 out via memory. Note however : the call to flush_page_to_ram in
911 (generic)/mm/memory.c:(break_cow) undoes all this good work in that one
912 very important case!
913
914 TBD : can we guarantee that on every call, any cache entries for
915 'from' are in the same colour sets as 'address' also? i.e. is this
916 always used just to deal with COW? (I suspect not). */
917
918 /* There are two possibilities here for when the page 'from' was last accessed:
919 * by the kernel : this is OK, no purge required.
920 * by the/a user (e.g. for break_COW) : need to purge.
921
922 If the potential user mapping at 'address' is the same colour as
923 'from' there is no need to purge any cache lines from the 'from'
924 page mapped into cache sets of colour 'address'. (The copy will be
925 accessing the page through 'from').
926 */
927
928 if (((address ^ (unsigned long) from) & CACHE_OC_SYN_MASK) != 0) {
929 sh64_dcache_purge_coloured_phy_page(__pa(from), address);
930 }
931
932 if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) {
933 /* No synonym problem on destination */
934 sh64_page_copy(from, to);
935 } else {
936 sh64_copy_user_page_coloured(to, from, address);
937 }
938
939 /* Note, don't need to flush 'from' page from the cache again - it's
940 done anyway by the generic code */
941}
942
943void clear_user_page(void *to, unsigned long address, struct page *page)
944{
945 /* 'to' is a kernel virtual address (within the superpage
946 mapping of the physical RAM). 'address' is the user virtual address
947 where the 'to' page will be mapped after. This allows a custom
948 mapping to be used to ensure that the new copy is placed in the
949 right cache sets for the user to see it without having to bounce it
950 out via memory.
951 */
952
953 if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) {
954 /* No synonym problem on destination */
955 sh64_page_clear(to);
956 } else {
957 sh64_clear_user_page_coloured(to, address);
958 }
959}
960
961#endif /* !CONFIG_DCACHE_DISABLED */
962
963/****************************************************************************/
964
965void flush_dcache_page(struct page *page)
966{
967 sh64_dcache_purge_phy_page(page_to_phys(page));
968 wmb();
969}
970
971/****************************************************************************/
972
973void flush_icache_range(unsigned long start, unsigned long end)
974{
975 /* Flush the range [start,end] of kernel virtual adddress space from
976 the I-cache. The corresponding range must be purged from the
977 D-cache also because the SH-5 doesn't have cache snooping between
978 the caches. The addresses will be visible through the superpage
979 mapping, therefore it's guaranteed that there no cache entries for
980 the range in cache sets of the wrong colour.
981
982 Primarily used for cohering the I-cache after a module has
983 been loaded. */
984
985 /* We also make sure to purge the same range from the D-cache since
986 flush_page_to_ram() won't be doing this for us! */
987
988 sh64_dcache_purge_kernel_range(start, end);
989 wmb();
990 sh64_icache_inv_kernel_range(start, end);
991}
992
993/****************************************************************************/
994
995void flush_icache_user_range(struct vm_area_struct *vma,
996 struct page *page, unsigned long addr, int len)
997{
998 /* Flush the range of user (defined by vma->vm_mm) address space
999 starting at 'addr' for 'len' bytes from the cache. The range does
1000 not straddle a page boundary, the unique physical page containing
1001 the range is 'page'. This seems to be used mainly for invalidating
1002 an address range following a poke into the program text through the
1003 ptrace() call from another process (e.g. for BRK instruction
1004 insertion). */
1005
1006 sh64_dcache_purge_coloured_phy_page(page_to_phys(page), addr);
1007 mb();
1008
1009 if (vma->vm_flags & VM_EXEC) {
1010 sh64_icache_inv_user_small_range(vma->vm_mm, addr, len);
1011 }
1012}
1013
1014/*##########################################################################
1015 ARCH/SH64 PRIVATE CALLABLE API.
1016 ##########################################################################*/
1017
1018void flush_cache_sigtramp(unsigned long start, unsigned long end)
1019{
1020 /* For the address range [start,end), write back the data from the
1021 D-cache and invalidate the corresponding region of the I-cache for
1022 the current process. Used to flush signal trampolines on the stack
1023 to make them executable. */
1024
1025 sh64_dcache_wback_current_user_range(start, end);
1026 wmb();
1027 sh64_icache_inv_current_user_range(start, end);
1028}
1029
diff --git a/arch/sh/mm/cache-sh7705.c b/arch/sh/mm/cache-sh7705.c
index 4896d7376926..22dacc778823 100644
--- a/arch/sh/mm/cache-sh7705.c
+++ b/arch/sh/mm/cache-sh7705.c
@@ -71,7 +71,7 @@ void flush_icache_range(unsigned long start, unsigned long end)
71/* 71/*
72 * Writeback&Invalidate the D-cache of the page 72 * Writeback&Invalidate the D-cache of the page
73 */ 73 */
74static void __flush_dcache_page(unsigned long phys) 74static void __uses_jump_to_uncached __flush_dcache_page(unsigned long phys)
75{ 75{
76 unsigned long ways, waysize, addrstart; 76 unsigned long ways, waysize, addrstart;
77 unsigned long flags; 77 unsigned long flags;
@@ -92,7 +92,7 @@ static void __flush_dcache_page(unsigned long phys)
92 * possible. 92 * possible.
93 */ 93 */
94 local_irq_save(flags); 94 local_irq_save(flags);
95 jump_to_P2(); 95 jump_to_uncached();
96 96
97 ways = current_cpu_data.dcache.ways; 97 ways = current_cpu_data.dcache.ways;
98 waysize = current_cpu_data.dcache.sets; 98 waysize = current_cpu_data.dcache.sets;
@@ -118,7 +118,7 @@ static void __flush_dcache_page(unsigned long phys)
118 addrstart += current_cpu_data.dcache.way_incr; 118 addrstart += current_cpu_data.dcache.way_incr;
119 } while (--ways); 119 } while (--ways);
120 120
121 back_to_P1(); 121 back_to_cached();
122 local_irq_restore(flags); 122 local_irq_restore(flags);
123} 123}
124 124
@@ -132,15 +132,15 @@ void flush_dcache_page(struct page *page)
132 __flush_dcache_page(PHYSADDR(page_address(page))); 132 __flush_dcache_page(PHYSADDR(page_address(page)));
133} 133}
134 134
135void flush_cache_all(void) 135void __uses_jump_to_uncached flush_cache_all(void)
136{ 136{
137 unsigned long flags; 137 unsigned long flags;
138 138
139 local_irq_save(flags); 139 local_irq_save(flags);
140 jump_to_P2(); 140 jump_to_uncached();
141 141
142 cache_wback_all(); 142 cache_wback_all();
143 back_to_P1(); 143 back_to_cached();
144 local_irq_restore(flags); 144 local_irq_restore(flags);
145} 145}
146 146
diff --git a/arch/sh/mm/clear_page.S b/arch/sh/mm/clear_page.S
deleted file mode 100644
index 7a7c81ee3f01..000000000000
--- a/arch/sh/mm/clear_page.S
+++ /dev/null
@@ -1,152 +0,0 @@
1/*
2 * __clear_user_page, __clear_user, clear_page implementation of SuperH
3 *
4 * Copyright (C) 2001 Kaz Kojima
5 * Copyright (C) 2001, 2002 Niibe Yutaka
6 * Copyright (C) 2006 Paul Mundt
7 */
8#include <linux/linkage.h>
9#include <asm/page.h>
10
11/*
12 * clear_page_slow
13 * @to: P1 address
14 *
15 * void clear_page_slow(void *to)
16 */
17
18/*
19 * r0 --- scratch
20 * r4 --- to
21 * r5 --- to + PAGE_SIZE
22 */
23ENTRY(clear_page_slow)
24 mov r4,r5
25 mov.l .Llimit,r0
26 add r0,r5
27 mov #0,r0
28 !
291:
30#if defined(CONFIG_CPU_SH3)
31 mov.l r0,@r4
32#elif defined(CONFIG_CPU_SH4)
33 movca.l r0,@r4
34 mov r4,r1
35#endif
36 add #32,r4
37 mov.l r0,@-r4
38 mov.l r0,@-r4
39 mov.l r0,@-r4
40 mov.l r0,@-r4
41 mov.l r0,@-r4
42 mov.l r0,@-r4
43 mov.l r0,@-r4
44#if defined(CONFIG_CPU_SH4)
45 ocbwb @r1
46#endif
47 cmp/eq r5,r4
48 bf/s 1b
49 add #28,r4
50 !
51 rts
52 nop
53.Llimit: .long (PAGE_SIZE-28)
54
55ENTRY(__clear_user)
56 !
57 mov #0, r0
58 mov #0xe0, r1 ! 0xffffffe0
59 !
60 ! r4..(r4+31)&~32 -------- not aligned [ Area 0 ]
61 ! (r4+31)&~32..(r4+r5)&~32 -------- aligned [ Area 1 ]
62 ! (r4+r5)&~32..r4+r5 -------- not aligned [ Area 2 ]
63 !
64 ! Clear area 0
65 mov r4, r2
66 !
67 tst r1, r5 ! length < 32
68 bt .Larea2 ! skip to remainder
69 !
70 add #31, r2
71 and r1, r2
72 cmp/eq r4, r2
73 bt .Larea1
74 mov r2, r3
75 sub r4, r3
76 mov r3, r7
77 mov r4, r2
78 !
79.L0: dt r3
800: mov.b r0, @r2
81 bf/s .L0
82 add #1, r2
83 !
84 sub r7, r5
85 mov r2, r4
86.Larea1:
87 mov r4, r3
88 add r5, r3
89 and r1, r3
90 cmp/hi r2, r3
91 bf .Larea2
92 !
93 ! Clear area 1
94#if defined(CONFIG_CPU_SH4)
951: movca.l r0, @r2
96#else
971: mov.l r0, @r2
98#endif
99 add #4, r2
1002: mov.l r0, @r2
101 add #4, r2
1023: mov.l r0, @r2
103 add #4, r2
1044: mov.l r0, @r2
105 add #4, r2
1065: mov.l r0, @r2
107 add #4, r2
1086: mov.l r0, @r2
109 add #4, r2
1107: mov.l r0, @r2
111 add #4, r2
1128: mov.l r0, @r2
113 add #4, r2
114 cmp/hi r2, r3
115 bt/s 1b
116 nop
117 !
118 ! Clear area 2
119.Larea2:
120 mov r4, r3
121 add r5, r3
122 cmp/hs r3, r2
123 bt/s .Ldone
124 sub r2, r3
125.L2: dt r3
1269: mov.b r0, @r2
127 bf/s .L2
128 add #1, r2
129 !
130.Ldone: rts
131 mov #0, r0 ! return 0 as normal return
132
133 ! return the number of bytes remained
134.Lbad_clear_user:
135 mov r4, r0
136 add r5, r0
137 rts
138 sub r2, r0
139
140.section __ex_table,"a"
141 .align 2
142 .long 0b, .Lbad_clear_user
143 .long 1b, .Lbad_clear_user
144 .long 2b, .Lbad_clear_user
145 .long 3b, .Lbad_clear_user
146 .long 4b, .Lbad_clear_user
147 .long 5b, .Lbad_clear_user
148 .long 6b, .Lbad_clear_user
149 .long 7b, .Lbad_clear_user
150 .long 8b, .Lbad_clear_user
151 .long 9b, .Lbad_clear_user
152.previous
diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c
index e220c29a3c00..7b2131c9eeda 100644
--- a/arch/sh/mm/consistent.c
+++ b/arch/sh/mm/consistent.c
@@ -1,7 +1,9 @@
1/* 1/*
2 * arch/sh/mm/consistent.c 2 * arch/sh/mm/consistent.c
3 * 3 *
4 * Copyright (C) 2004 Paul Mundt 4 * Copyright (C) 2004 - 2007 Paul Mundt
5 *
6 * Declared coherent memory functions based on arch/x86/kernel/pci-dma_32.c
5 * 7 *
6 * This file is subject to the terms and conditions of the GNU General Public 8 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file "COPYING" in the main directory of this archive 9 * License. See the file "COPYING" in the main directory of this archive
@@ -13,58 +15,152 @@
13#include <asm/addrspace.h> 15#include <asm/addrspace.h>
14#include <asm/io.h> 16#include <asm/io.h>
15 17
16void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle) 18struct dma_coherent_mem {
19 void *virt_base;
20 u32 device_base;
21 int size;
22 int flags;
23 unsigned long *bitmap;
24};
25
26void *dma_alloc_coherent(struct device *dev, size_t size,
27 dma_addr_t *dma_handle, gfp_t gfp)
17{ 28{
18 struct page *page, *end, *free;
19 void *ret; 29 void *ret;
20 int order; 30 struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
31 int order = get_order(size);
21 32
22 size = PAGE_ALIGN(size); 33 if (mem) {
23 order = get_order(size); 34 int page = bitmap_find_free_region(mem->bitmap, mem->size,
35 order);
36 if (page >= 0) {
37 *dma_handle = mem->device_base + (page << PAGE_SHIFT);
38 ret = mem->virt_base + (page << PAGE_SHIFT);
39 memset(ret, 0, size);
40 return ret;
41 }
42 if (mem->flags & DMA_MEMORY_EXCLUSIVE)
43 return NULL;
44 }
24 45
25 page = alloc_pages(gfp, order); 46 ret = (void *)__get_free_pages(gfp, order);
26 if (!page)
27 return NULL;
28 split_page(page, order);
29 47
30 ret = page_address(page); 48 if (ret != NULL) {
31 memset(ret, 0, size); 49 memset(ret, 0, size);
32 *handle = virt_to_phys(ret); 50 /*
51 * Pages from the page allocator may have data present in
52 * cache. So flush the cache before using uncached memory.
53 */
54 dma_cache_sync(NULL, ret, size, DMA_BIDIRECTIONAL);
55 *dma_handle = virt_to_phys(ret);
56 }
57 return ret;
58}
59EXPORT_SYMBOL(dma_alloc_coherent);
33 60
34 /* 61void dma_free_coherent(struct device *dev, size_t size,
35 * We must flush the cache before we pass it on to the device 62 void *vaddr, dma_addr_t dma_handle)
36 */ 63{
37 __flush_purge_region(ret, size); 64 struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
65 int order = get_order(size);
38 66
39 page = virt_to_page(ret); 67 if (mem && vaddr >= mem->virt_base && vaddr < (mem->virt_base + (mem->size << PAGE_SHIFT))) {
40 free = page + (size >> PAGE_SHIFT); 68 int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
41 end = page + (1 << order);
42 69
43 while (++page < end) { 70 bitmap_release_region(mem->bitmap, page, order);
44 /* Free any unused pages */ 71 } else {
45 if (page >= free) { 72 WARN_ON(irqs_disabled()); /* for portability */
46 __free_page(page); 73 BUG_ON(mem && mem->flags & DMA_MEMORY_EXCLUSIVE);
47 } 74 free_pages((unsigned long)vaddr, order);
48 } 75 }
76}
77EXPORT_SYMBOL(dma_free_coherent);
78
79int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
80 dma_addr_t device_addr, size_t size, int flags)
81{
82 void __iomem *mem_base = NULL;
83 int pages = size >> PAGE_SHIFT;
84 int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
85
86 if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
87 goto out;
88 if (!size)
89 goto out;
90 if (dev->dma_mem)
91 goto out;
92
93 /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
49 94
50 return P2SEGADDR(ret); 95 mem_base = ioremap_nocache(bus_addr, size);
96 if (!mem_base)
97 goto out;
98
99 dev->dma_mem = kmalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
100 if (!dev->dma_mem)
101 goto out;
102 dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
103 if (!dev->dma_mem->bitmap)
104 goto free1_out;
105
106 dev->dma_mem->virt_base = mem_base;
107 dev->dma_mem->device_base = device_addr;
108 dev->dma_mem->size = pages;
109 dev->dma_mem->flags = flags;
110
111 if (flags & DMA_MEMORY_MAP)
112 return DMA_MEMORY_MAP;
113
114 return DMA_MEMORY_IO;
115
116 free1_out:
117 kfree(dev->dma_mem);
118 out:
119 if (mem_base)
120 iounmap(mem_base);
121 return 0;
51} 122}
123EXPORT_SYMBOL(dma_declare_coherent_memory);
52 124
53void consistent_free(void *vaddr, size_t size) 125void dma_release_declared_memory(struct device *dev)
54{ 126{
55 unsigned long addr = P1SEGADDR((unsigned long)vaddr); 127 struct dma_coherent_mem *mem = dev->dma_mem;
56 struct page *page=virt_to_page(addr);
57 int num_pages=(size+PAGE_SIZE-1) >> PAGE_SHIFT;
58 int i;
59 128
60 for(i=0;i<num_pages;i++) { 129 if (!mem)
61 __free_page((page+i)); 130 return;
62 } 131 dev->dma_mem = NULL;
132 iounmap(mem->virt_base);
133 kfree(mem->bitmap);
134 kfree(mem);
63} 135}
136EXPORT_SYMBOL(dma_release_declared_memory);
64 137
65void consistent_sync(void *vaddr, size_t size, int direction) 138void *dma_mark_declared_memory_occupied(struct device *dev,
139 dma_addr_t device_addr, size_t size)
66{ 140{
67 void * p1addr = (void*) P1SEGADDR((unsigned long)vaddr); 141 struct dma_coherent_mem *mem = dev->dma_mem;
142 int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT;
143 int pos, err;
144
145 if (!mem)
146 return ERR_PTR(-EINVAL);
147
148 pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
149 err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
150 if (err != 0)
151 return ERR_PTR(err);
152 return mem->virt_base + (pos << PAGE_SHIFT);
153}
154EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
155
156void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
157 enum dma_data_direction direction)
158{
159#ifdef CONFIG_CPU_SH5
160 void *p1addr = vaddr;
161#else
162 void *p1addr = (void*) P1SEGADDR((unsigned long)vaddr);
163#endif
68 164
69 switch (direction) { 165 switch (direction) {
70 case DMA_FROM_DEVICE: /* invalidate only */ 166 case DMA_FROM_DEVICE: /* invalidate only */
@@ -80,8 +176,4 @@ void consistent_sync(void *vaddr, size_t size, int direction)
80 BUG(); 176 BUG();
81 } 177 }
82} 178}
83 179EXPORT_SYMBOL(dma_cache_sync);
84EXPORT_SYMBOL(consistent_alloc);
85EXPORT_SYMBOL(consistent_free);
86EXPORT_SYMBOL(consistent_sync);
87
diff --git a/arch/sh/mm/copy_page.S b/arch/sh/mm/copy_page.S
deleted file mode 100644
index 40685018b952..000000000000
--- a/arch/sh/mm/copy_page.S
+++ /dev/null
@@ -1,388 +0,0 @@
1/*
2 * copy_page, __copy_user_page, __copy_user implementation of SuperH
3 *
4 * Copyright (C) 2001 Niibe Yutaka & Kaz Kojima
5 * Copyright (C) 2002 Toshinobu Sugioka
6 * Copyright (C) 2006 Paul Mundt
7 */
8#include <linux/linkage.h>
9#include <asm/page.h>
10
11/*
12 * copy_page_slow
13 * @to: P1 address
14 * @from: P1 address
15 *
16 * void copy_page_slow(void *to, void *from)
17 */
18
19/*
20 * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch
21 * r8 --- from + PAGE_SIZE
22 * r9 --- not used
23 * r10 --- to
24 * r11 --- from
25 */
26ENTRY(copy_page_slow)
27 mov.l r8,@-r15
28 mov.l r10,@-r15
29 mov.l r11,@-r15
30 mov r4,r10
31 mov r5,r11
32 mov r5,r8
33 mov.l .Lpsz,r0
34 add r0,r8
35 !
361: mov.l @r11+,r0
37 mov.l @r11+,r1
38 mov.l @r11+,r2
39 mov.l @r11+,r3
40 mov.l @r11+,r4
41 mov.l @r11+,r5
42 mov.l @r11+,r6
43 mov.l @r11+,r7
44#if defined(CONFIG_CPU_SH3)
45 mov.l r0,@r10
46#elif defined(CONFIG_CPU_SH4)
47 movca.l r0,@r10
48 mov r10,r0
49#endif
50 add #32,r10
51 mov.l r7,@-r10
52 mov.l r6,@-r10
53 mov.l r5,@-r10
54 mov.l r4,@-r10
55 mov.l r3,@-r10
56 mov.l r2,@-r10
57 mov.l r1,@-r10
58#if defined(CONFIG_CPU_SH4)
59 ocbwb @r0
60#endif
61 cmp/eq r11,r8
62 bf/s 1b
63 add #28,r10
64 !
65 mov.l @r15+,r11
66 mov.l @r15+,r10
67 mov.l @r15+,r8
68 rts
69 nop
70
71 .align 2
72.Lpsz: .long PAGE_SIZE
73/*
74 * __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n);
75 * Return the number of bytes NOT copied
76 */
77#define EX(...) \
78 9999: __VA_ARGS__ ; \
79 .section __ex_table, "a"; \
80 .long 9999b, 6000f ; \
81 .previous
82ENTRY(__copy_user)
83 ! Check if small number of bytes
84 mov #11,r0
85 mov r4,r3
86 cmp/gt r0,r6 ! r6 (len) > r0 (11)
87 bf/s .L_cleanup_loop_no_pop
88 add r6,r3 ! last destination address
89
90 ! Calculate bytes needed to align to src
91 mov.l r11,@-r15
92 neg r5,r0
93 mov.l r10,@-r15
94 add #4,r0
95 mov.l r9,@-r15
96 and #3,r0
97 mov.l r8,@-r15
98 tst r0,r0
99 bt 2f
100
1011:
102 ! Copy bytes to long word align src
103EX( mov.b @r5+,r1 )
104 dt r0
105 add #-1,r6
106EX( mov.b r1,@r4 )
107 bf/s 1b
108 add #1,r4
109
110 ! Jump to appropriate routine depending on dest
1112: mov #3,r1
112 mov r6, r2
113 and r4,r1
114 shlr2 r2
115 shll2 r1
116 mova .L_jump_tbl,r0
117 mov.l @(r0,r1),r1
118 jmp @r1
119 nop
120
121 .align 2
122.L_jump_tbl:
123 .long .L_dest00
124 .long .L_dest01
125 .long .L_dest10
126 .long .L_dest11
127
128/*
129 * Come here if there are less than 12 bytes to copy
130 *
131 * Keep the branch target close, so the bf/s callee doesn't overflow
132 * and result in a more expensive branch being inserted. This is the
133 * fast-path for small copies, the jump via the jump table will hit the
134 * default slow-path cleanup. -PFM.
135 */
136.L_cleanup_loop_no_pop:
137 tst r6,r6 ! Check explicitly for zero
138 bt 1f
139
1402:
141EX( mov.b @r5+,r0 )
142 dt r6
143EX( mov.b r0,@r4 )
144 bf/s 2b
145 add #1,r4
146
1471: mov #0,r0 ! normal return
1485000:
149
150# Exception handler:
151.section .fixup, "ax"
1526000:
153 mov.l 8000f,r1
154 mov r3,r0
155 jmp @r1
156 sub r4,r0
157 .align 2
1588000: .long 5000b
159
160.previous
161 rts
162 nop
163
164! Destination = 00
165
166.L_dest00:
167 ! Skip the large copy for small transfers
168 mov #(32+32-4), r0
169 cmp/gt r6, r0 ! r0 (60) > r6 (len)
170 bt 1f
171
172 ! Align dest to a 32 byte boundary
173 neg r4,r0
174 add #0x20, r0
175 and #0x1f, r0
176 tst r0, r0
177 bt 2f
178
179 sub r0, r6
180 shlr2 r0
1813:
182EX( mov.l @r5+,r1 )
183 dt r0
184EX( mov.l r1,@r4 )
185 bf/s 3b
186 add #4,r4
187
1882:
189EX( mov.l @r5+,r0 )
190EX( mov.l @r5+,r1 )
191EX( mov.l @r5+,r2 )
192EX( mov.l @r5+,r7 )
193EX( mov.l @r5+,r8 )
194EX( mov.l @r5+,r9 )
195EX( mov.l @r5+,r10 )
196EX( mov.l @r5+,r11 )
197#ifdef CONFIG_CPU_SH4
198EX( movca.l r0,@r4 )
199#else
200EX( mov.l r0,@r4 )
201#endif
202 add #-32, r6
203EX( mov.l r1,@(4,r4) )
204 mov #32, r0
205EX( mov.l r2,@(8,r4) )
206 cmp/gt r6, r0 ! r0 (32) > r6 (len)
207EX( mov.l r7,@(12,r4) )
208EX( mov.l r8,@(16,r4) )
209EX( mov.l r9,@(20,r4) )
210EX( mov.l r10,@(24,r4) )
211EX( mov.l r11,@(28,r4) )
212 bf/s 2b
213 add #32,r4
214
2151: mov r6, r0
216 shlr2 r0
217 tst r0, r0
218 bt .L_cleanup
2191:
220EX( mov.l @r5+,r1 )
221 dt r0
222EX( mov.l r1,@r4 )
223 bf/s 1b
224 add #4,r4
225
226 bra .L_cleanup
227 nop
228
229! Destination = 10
230
231.L_dest10:
232 mov r2,r7
233 shlr2 r7
234 shlr r7
235 tst r7,r7
236 mov #7,r0
237 bt/s 1f
238 and r0,r2
2392:
240 dt r7
241#ifdef CONFIG_CPU_LITTLE_ENDIAN
242EX( mov.l @r5+,r0 )
243EX( mov.l @r5+,r1 )
244EX( mov.l @r5+,r8 )
245EX( mov.l @r5+,r9 )
246EX( mov.l @r5+,r10 )
247EX( mov.w r0,@r4 )
248 add #2,r4
249 xtrct r1,r0
250 xtrct r8,r1
251 xtrct r9,r8
252 xtrct r10,r9
253
254EX( mov.l r0,@r4 )
255EX( mov.l r1,@(4,r4) )
256EX( mov.l r8,@(8,r4) )
257EX( mov.l r9,@(12,r4) )
258
259EX( mov.l @r5+,r1 )
260EX( mov.l @r5+,r8 )
261EX( mov.l @r5+,r0 )
262 xtrct r1,r10
263 xtrct r8,r1
264 xtrct r0,r8
265 shlr16 r0
266EX( mov.l r10,@(16,r4) )
267EX( mov.l r1,@(20,r4) )
268EX( mov.l r8,@(24,r4) )
269EX( mov.w r0,@(28,r4) )
270 bf/s 2b
271 add #30,r4
272#else
273EX( mov.l @(28,r5),r0 )
274EX( mov.l @(24,r5),r8 )
275EX( mov.l @(20,r5),r9 )
276EX( mov.l @(16,r5),r10 )
277EX( mov.w r0,@(30,r4) )
278 add #-2,r4
279 xtrct r8,r0
280 xtrct r9,r8
281 xtrct r10,r9
282EX( mov.l r0,@(28,r4) )
283EX( mov.l r8,@(24,r4) )
284EX( mov.l r9,@(20,r4) )
285
286EX( mov.l @(12,r5),r0 )
287EX( mov.l @(8,r5),r8 )
288 xtrct r0,r10
289EX( mov.l @(4,r5),r9 )
290 mov.l r10,@(16,r4)
291EX( mov.l @r5,r10 )
292 xtrct r8,r0
293 xtrct r9,r8
294 xtrct r10,r9
295EX( mov.l r0,@(12,r4) )
296EX( mov.l r8,@(8,r4) )
297 swap.w r10,r0
298EX( mov.l r9,@(4,r4) )
299EX( mov.w r0,@(2,r4) )
300
301 add #32,r5
302 bf/s 2b
303 add #34,r4
304#endif
305 tst r2,r2
306 bt .L_cleanup
307
3081: ! Read longword, write two words per iteration
309EX( mov.l @r5+,r0 )
310 dt r2
311#ifdef CONFIG_CPU_LITTLE_ENDIAN
312EX( mov.w r0,@r4 )
313 shlr16 r0
314EX( mov.w r0,@(2,r4) )
315#else
316EX( mov.w r0,@(2,r4) )
317 shlr16 r0
318EX( mov.w r0,@r4 )
319#endif
320 bf/s 1b
321 add #4,r4
322
323 bra .L_cleanup
324 nop
325
326! Destination = 01 or 11
327
328.L_dest01:
329.L_dest11:
330 ! Read longword, write byte, word, byte per iteration
331EX( mov.l @r5+,r0 )
332 dt r2
333#ifdef CONFIG_CPU_LITTLE_ENDIAN
334EX( mov.b r0,@r4 )
335 shlr8 r0
336 add #1,r4
337EX( mov.w r0,@r4 )
338 shlr16 r0
339EX( mov.b r0,@(2,r4) )
340 bf/s .L_dest01
341 add #3,r4
342#else
343EX( mov.b r0,@(3,r4) )
344 shlr8 r0
345 swap.w r0,r7
346EX( mov.b r7,@r4 )
347 add #1,r4
348EX( mov.w r0,@r4 )
349 bf/s .L_dest01
350 add #3,r4
351#endif
352
353! Cleanup last few bytes
354.L_cleanup:
355 mov r6,r0
356 and #3,r0
357 tst r0,r0
358 bt .L_exit
359 mov r0,r6
360
361.L_cleanup_loop:
362EX( mov.b @r5+,r0 )
363 dt r6
364EX( mov.b r0,@r4 )
365 bf/s .L_cleanup_loop
366 add #1,r4
367
368.L_exit:
369 mov #0,r0 ! normal return
370
3715000:
372
373# Exception handler:
374.section .fixup, "ax"
3756000:
376 mov.l 8000f,r1
377 mov r3,r0
378 jmp @r1
379 sub r4,r0
380 .align 2
3818000: .long 5000b
382
383.previous
384 mov.l @r15+,r8
385 mov.l @r15+,r9
386 mov.l @r15+,r10
387 rts
388 mov.l @r15+,r11
diff --git a/arch/sh/mm/extable.c b/arch/sh/mm/extable_32.c
index c1cf4463d09d..c1cf4463d09d 100644
--- a/arch/sh/mm/extable.c
+++ b/arch/sh/mm/extable_32.c
diff --git a/arch/sh/mm/extable_64.c b/arch/sh/mm/extable_64.c
new file mode 100644
index 000000000000..f05499688d88
--- /dev/null
+++ b/arch/sh/mm/extable_64.c
@@ -0,0 +1,82 @@
1/*
2 * arch/sh/mm/extable_64.c
3 *
4 * Copyright (C) 2003 Richard Curnow
5 * Copyright (C) 2003, 2004 Paul Mundt
6 *
7 * Cloned from the 2.5 SH version..
8 *
9 * This file is subject to the terms and conditions of the GNU General Public
10 * License. See the file "COPYING" in the main directory of this archive
11 * for more details.
12 */
13#include <linux/rwsem.h>
14#include <linux/module.h>
15#include <asm/uaccess.h>
16
17extern unsigned long copy_user_memcpy, copy_user_memcpy_end;
18extern void __copy_user_fixup(void);
19
20static const struct exception_table_entry __copy_user_fixup_ex = {
21 .fixup = (unsigned long)&__copy_user_fixup,
22};
23
24/*
25 * Some functions that may trap due to a bad user-mode address have too
26 * many loads and stores in them to make it at all practical to label
27 * each one and put them all in the main exception table.
28 *
29 * In particular, the fast memcpy routine is like this. It's fix-up is
30 * just to fall back to a slow byte-at-a-time copy, which is handled the
31 * conventional way. So it's functionally OK to just handle any trap
32 * occurring in the fast memcpy with that fixup.
33 */
34static const struct exception_table_entry *check_exception_ranges(unsigned long addr)
35{
36 if ((addr >= (unsigned long)&copy_user_memcpy) &&
37 (addr <= (unsigned long)&copy_user_memcpy_end))
38 return &__copy_user_fixup_ex;
39
40 return NULL;
41}
42
43/* Simple binary search */
44const struct exception_table_entry *
45search_extable(const struct exception_table_entry *first,
46 const struct exception_table_entry *last,
47 unsigned long value)
48{
49 const struct exception_table_entry *mid;
50
51 mid = check_exception_ranges(value);
52 if (mid)
53 return mid;
54
55 while (first <= last) {
56 long diff;
57
58 mid = (last - first) / 2 + first;
59 diff = mid->insn - value;
60 if (diff == 0)
61 return mid;
62 else if (diff < 0)
63 first = mid+1;
64 else
65 last = mid-1;
66 }
67
68 return NULL;
69}
70
71int fixup_exception(struct pt_regs *regs)
72{
73 const struct exception_table_entry *fixup;
74
75 fixup = search_exception_tables(regs->pc);
76 if (fixup) {
77 regs->pc = fixup->fixup;
78 return 1;
79 }
80
81 return 0;
82}
diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault_32.c
index 60d74f793a1d..33b43d20e9f6 100644
--- a/arch/sh/mm/fault.c
+++ b/arch/sh/mm/fault_32.c
@@ -172,7 +172,7 @@ no_context:
172 bust_spinlocks(1); 172 bust_spinlocks(1);
173 173
174 if (oops_may_print()) { 174 if (oops_may_print()) {
175 __typeof__(pte_val(__pte(0))) page; 175 unsigned long page;
176 176
177 if (address < PAGE_SIZE) 177 if (address < PAGE_SIZE)
178 printk(KERN_ALERT "Unable to handle kernel NULL " 178 printk(KERN_ALERT "Unable to handle kernel NULL "
diff --git a/arch/sh/mm/fault_64.c b/arch/sh/mm/fault_64.c
new file mode 100644
index 000000000000..399d53710d2f
--- /dev/null
+++ b/arch/sh/mm/fault_64.c
@@ -0,0 +1,275 @@
1/*
2 * The SH64 TLB miss.
3 *
4 * Original code from fault.c
5 * Copyright (C) 2000, 2001 Paolo Alberelli
6 *
7 * Fast PTE->TLB refill path
8 * Copyright (C) 2003 Richard.Curnow@superh.com
9 *
10 * IMPORTANT NOTES :
11 * The do_fast_page_fault function is called from a context in entry.S
12 * where very few registers have been saved. In particular, the code in
13 * this file must be compiled not to use ANY caller-save registers that
14 * are not part of the restricted save set. Also, it means that code in
15 * this file must not make calls to functions elsewhere in the kernel, or
16 * else the excepting context will see corruption in its caller-save
17 * registers. Plus, the entry.S save area is non-reentrant, so this code
18 * has to run with SR.BL==1, i.e. no interrupts taken inside it and panic
19 * on any exception.
20 *
21 * This file is subject to the terms and conditions of the GNU General Public
22 * License. See the file "COPYING" in the main directory of this archive
23 * for more details.
24 */
25#include <linux/signal.h>
26#include <linux/sched.h>
27#include <linux/kernel.h>
28#include <linux/errno.h>
29#include <linux/string.h>
30#include <linux/types.h>
31#include <linux/ptrace.h>
32#include <linux/mman.h>
33#include <linux/mm.h>
34#include <linux/smp.h>
35#include <linux/interrupt.h>
36#include <asm/system.h>
37#include <asm/tlb.h>
38#include <asm/io.h>
39#include <asm/uaccess.h>
40#include <asm/pgalloc.h>
41#include <asm/mmu_context.h>
42#include <asm/cpu/registers.h>
43
44/* Callable from fault.c, so not static */
45inline void __do_tlb_refill(unsigned long address,
46 unsigned long long is_text_not_data, pte_t *pte)
47{
48 unsigned long long ptel;
49 unsigned long long pteh=0;
50 struct tlb_info *tlbp;
51 unsigned long long next;
52
53 /* Get PTEL first */
54 ptel = pte_val(*pte);
55
56 /*
57 * Set PTEH register
58 */
59 pteh = address & MMU_VPN_MASK;
60
61 /* Sign extend based on neff. */
62#if (NEFF == 32)
63 /* Faster sign extension */
64 pteh = (unsigned long long)(signed long long)(signed long)pteh;
65#else
66 /* General case */
67 pteh = (pteh & NEFF_SIGN) ? (pteh | NEFF_MASK) : pteh;
68#endif
69
70 /* Set the ASID. */
71 pteh |= get_asid() << PTEH_ASID_SHIFT;
72 pteh |= PTEH_VALID;
73
74 /* Set PTEL register, set_pte has performed the sign extension */
75 ptel &= _PAGE_FLAGS_HARDWARE_MASK; /* drop software flags */
76
77 tlbp = is_text_not_data ? &(cpu_data->itlb) : &(cpu_data->dtlb);
78 next = tlbp->next;
79 __flush_tlb_slot(next);
80 asm volatile ("putcfg %0,1,%2\n\n\t"
81 "putcfg %0,0,%1\n"
82 : : "r" (next), "r" (pteh), "r" (ptel) );
83
84 next += TLB_STEP;
85 if (next > tlbp->last) next = tlbp->first;
86 tlbp->next = next;
87
88}
89
90static int handle_vmalloc_fault(struct mm_struct *mm,
91 unsigned long protection_flags,
92 unsigned long long textaccess,
93 unsigned long address)
94{
95 pgd_t *dir;
96 pud_t *pud;
97 pmd_t *pmd;
98 static pte_t *pte;
99 pte_t entry;
100
101 dir = pgd_offset_k(address);
102
103 pud = pud_offset(dir, address);
104 if (pud_none_or_clear_bad(pud))
105 return 0;
106
107 pmd = pmd_offset(pud, address);
108 if (pmd_none_or_clear_bad(pmd))
109 return 0;
110
111 pte = pte_offset_kernel(pmd, address);
112 entry = *pte;
113
114 if (pte_none(entry) || !pte_present(entry))
115 return 0;
116 if ((pte_val(entry) & protection_flags) != protection_flags)
117 return 0;
118
119 __do_tlb_refill(address, textaccess, pte);
120
121 return 1;
122}
123
124static int handle_tlbmiss(struct mm_struct *mm,
125 unsigned long long protection_flags,
126 unsigned long long textaccess,
127 unsigned long address)
128{
129 pgd_t *dir;
130 pud_t *pud;
131 pmd_t *pmd;
132 pte_t *pte;
133 pte_t entry;
134
135 /* NB. The PGD currently only contains a single entry - there is no
136 page table tree stored for the top half of the address space since
137 virtual pages in that region should never be mapped in user mode.
138 (In kernel mode, the only things in that region are the 512Mb super
139 page (locked in), and vmalloc (modules) + I/O device pages (handled
140 by handle_vmalloc_fault), so no PGD for the upper half is required
141 by kernel mode either).
142
143 See how mm->pgd is allocated and initialised in pgd_alloc to see why
144 the next test is necessary. - RPC */
145 if (address >= (unsigned long) TASK_SIZE)
146 /* upper half - never has page table entries. */
147 return 0;
148
149 dir = pgd_offset(mm, address);
150 if (pgd_none(*dir) || !pgd_present(*dir))
151 return 0;
152 if (!pgd_present(*dir))
153 return 0;
154
155 pud = pud_offset(dir, address);
156 if (pud_none(*pud) || !pud_present(*pud))
157 return 0;
158
159 pmd = pmd_offset(pud, address);
160 if (pmd_none(*pmd) || !pmd_present(*pmd))
161 return 0;
162
163 pte = pte_offset_kernel(pmd, address);
164 entry = *pte;
165
166 if (pte_none(entry) || !pte_present(entry))
167 return 0;
168
169 /*
170 * If the page doesn't have sufficient protection bits set to
171 * service the kind of fault being handled, there's not much
172 * point doing the TLB refill. Punt the fault to the general
173 * handler.
174 */
175 if ((pte_val(entry) & protection_flags) != protection_flags)
176 return 0;
177
178 __do_tlb_refill(address, textaccess, pte);
179
180 return 1;
181}
182
183/*
184 * Put all this information into one structure so that everything is just
185 * arithmetic relative to a single base address. This reduces the number
186 * of movi/shori pairs needed just to load addresses of static data.
187 */
188struct expevt_lookup {
189 unsigned short protection_flags[8];
190 unsigned char is_text_access[8];
191 unsigned char is_write_access[8];
192};
193
194#define PRU (1<<9)
195#define PRW (1<<8)
196#define PRX (1<<7)
197#define PRR (1<<6)
198
199#define DIRTY (_PAGE_DIRTY | _PAGE_ACCESSED)
200#define YOUNG (_PAGE_ACCESSED)
201
202/* Sized as 8 rather than 4 to allow checking the PTE's PRU bit against whether
203 the fault happened in user mode or privileged mode. */
204static struct expevt_lookup expevt_lookup_table = {
205 .protection_flags = {PRX, PRX, 0, 0, PRR, PRR, PRW, PRW},
206 .is_text_access = {1, 1, 0, 0, 0, 0, 0, 0}
207};
208
209/*
210 This routine handles page faults that can be serviced just by refilling a
211 TLB entry from an existing page table entry. (This case represents a very
212 large majority of page faults.) Return 1 if the fault was successfully
213 handled. Return 0 if the fault could not be handled. (This leads into the
214 general fault handling in fault.c which deals with mapping file-backed
215 pages, stack growth, segmentation faults, swapping etc etc)
216 */
217asmlinkage int do_fast_page_fault(unsigned long long ssr_md,
218 unsigned long long expevt,
219 unsigned long address)
220{
221 struct task_struct *tsk;
222 struct mm_struct *mm;
223 unsigned long long textaccess;
224 unsigned long long protection_flags;
225 unsigned long long index;
226 unsigned long long expevt4;
227
228 /* The next few lines implement a way of hashing EXPEVT into a
229 * small array index which can be used to lookup parameters
230 * specific to the type of TLBMISS being handled.
231 *
232 * Note:
233 * ITLBMISS has EXPEVT==0xa40
234 * RTLBMISS has EXPEVT==0x040
235 * WTLBMISS has EXPEVT==0x060
236 */
237 expevt4 = (expevt >> 4);
238 /* TODO : xor ssr_md into this expression too. Then we can check
239 * that PRU is set when it needs to be. */
240 index = expevt4 ^ (expevt4 >> 5);
241 index &= 7;
242 protection_flags = expevt_lookup_table.protection_flags[index];
243 textaccess = expevt_lookup_table.is_text_access[index];
244
245 /* SIM
246 * Note this is now called with interrupts still disabled
247 * This is to cope with being called for a missing IO port
248 * address with interrupts disabled. This should be fixed as
249 * soon as we have a better 'fast path' miss handler.
250 *
251 * Plus take care how you try and debug this stuff.
252 * For example, writing debug data to a port which you
253 * have just faulted on is not going to work.
254 */
255
256 tsk = current;
257 mm = tsk->mm;
258
259 if ((address >= VMALLOC_START && address < VMALLOC_END) ||
260 (address >= IOBASE_VADDR && address < IOBASE_END)) {
261 if (ssr_md)
262 /*
263 * Process-contexts can never have this address
264 * range mapped
265 */
266 if (handle_vmalloc_fault(mm, protection_flags,
267 textaccess, address))
268 return 1;
269 } else if (!in_interrupt() && mm) {
270 if (handle_tlbmiss(mm, protection_flags, textaccess, address))
271 return 1;
272 }
273
274 return 0;
275}
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index d5e160da64b2..2918c6b14659 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -23,9 +23,7 @@
23 23
24DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 24DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
25pgd_t swapper_pg_dir[PTRS_PER_PGD]; 25pgd_t swapper_pg_dir[PTRS_PER_PGD];
26 26unsigned long cached_to_uncached = 0;
27void (*copy_page)(void *from, void *to);
28void (*clear_page)(void *to);
29 27
30void show_mem(void) 28void show_mem(void)
31{ 29{
@@ -102,7 +100,8 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot)
102 100
103 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot)); 101 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot));
104 102
105 flush_tlb_one(get_asid(), addr); 103 if (cached_to_uncached)
104 flush_tlb_one(get_asid(), addr);
106} 105}
107 106
108/* 107/*
@@ -131,6 +130,37 @@ void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
131 130
132 set_pte_phys(address, phys, prot); 131 set_pte_phys(address, phys, prot);
133} 132}
133
134void __init page_table_range_init(unsigned long start, unsigned long end,
135 pgd_t *pgd_base)
136{
137 pgd_t *pgd;
138 pud_t *pud;
139 pmd_t *pmd;
140 int pgd_idx;
141 unsigned long vaddr;
142
143 vaddr = start & PMD_MASK;
144 end = (end + PMD_SIZE - 1) & PMD_MASK;
145 pgd_idx = pgd_index(vaddr);
146 pgd = pgd_base + pgd_idx;
147
148 for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
149 BUG_ON(pgd_none(*pgd));
150 pud = pud_offset(pgd, 0);
151 BUG_ON(pud_none(*pud));
152 pmd = pmd_offset(pud, 0);
153
154 if (!pmd_present(*pmd)) {
155 pte_t *pte_table;
156 pte_table = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
157 memset(pte_table, 0, PAGE_SIZE);
158 pmd_populate_kernel(&init_mm, pmd, pte_table);
159 }
160
161 vaddr += PMD_SIZE;
162 }
163}
134#endif /* CONFIG_MMU */ 164#endif /* CONFIG_MMU */
135 165
136/* 166/*
@@ -150,6 +180,11 @@ void __init paging_init(void)
150 * check for a null value. */ 180 * check for a null value. */
151 set_TTB(swapper_pg_dir); 181 set_TTB(swapper_pg_dir);
152 182
183 /* Populate the relevant portions of swapper_pg_dir so that
184 * we can use the fixmap entries without calling kmalloc.
185 * pte's will be filled in by __set_fixmap(). */
186 page_table_range_init(FIXADDR_START, FIXADDR_TOP, swapper_pg_dir);
187
153 memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 188 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
154 189
155 for_each_online_node(nid) { 190 for_each_online_node(nid) {
@@ -167,9 +202,22 @@ void __init paging_init(void)
167 } 202 }
168 203
169 free_area_init_nodes(max_zone_pfns); 204 free_area_init_nodes(max_zone_pfns);
205
206 /* Set up the uncached fixmap */
207 set_fixmap_nocache(FIX_UNCACHED, __pa(&__uncached_start));
208
209#ifdef CONFIG_29BIT
210 /*
211 * Handle trivial transitions between cached and uncached
212 * segments, making use of the 1:1 mapping relationship in
213 * 512MB lowmem.
214 */
215 cached_to_uncached = P2SEG - P1SEG;
216#endif
170} 217}
171 218
172static struct kcore_list kcore_mem, kcore_vmalloc; 219static struct kcore_list kcore_mem, kcore_vmalloc;
220int after_bootmem = 0;
173 221
174void __init mem_init(void) 222void __init mem_init(void)
175{ 223{
@@ -202,17 +250,7 @@ void __init mem_init(void)
202 memset(empty_zero_page, 0, PAGE_SIZE); 250 memset(empty_zero_page, 0, PAGE_SIZE);
203 __flush_wback_region(empty_zero_page, PAGE_SIZE); 251 __flush_wback_region(empty_zero_page, PAGE_SIZE);
204 252
205 /* 253 after_bootmem = 1;
206 * Setup wrappers for copy/clear_page(), these will get overridden
207 * later in the boot process if a better method is available.
208 */
209#ifdef CONFIG_MMU
210 copy_page = copy_page_slow;
211 clear_page = clear_page_slow;
212#else
213 copy_page = copy_page_nommu;
214 clear_page = clear_page_nommu;
215#endif
216 254
217 codesize = (unsigned long) &_etext - (unsigned long) &_text; 255 codesize = (unsigned long) &_etext - (unsigned long) &_text;
218 datasize = (unsigned long) &_edata - (unsigned long) &_etext; 256 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
diff --git a/arch/sh/mm/ioremap.c b/arch/sh/mm/ioremap_32.c
index 0c7b7e33abdc..0c7b7e33abdc 100644
--- a/arch/sh/mm/ioremap.c
+++ b/arch/sh/mm/ioremap_32.c
diff --git a/arch/sh/mm/ioremap_64.c b/arch/sh/mm/ioremap_64.c
new file mode 100644
index 000000000000..e27d16519235
--- /dev/null
+++ b/arch/sh/mm/ioremap_64.c
@@ -0,0 +1,404 @@
1/*
2 * arch/sh/mm/ioremap_64.c
3 *
4 * Copyright (C) 2000, 2001 Paolo Alberelli
5 * Copyright (C) 2003 - 2007 Paul Mundt
6 *
7 * Mostly derived from arch/sh/mm/ioremap.c which, in turn is mostly
8 * derived from arch/i386/mm/ioremap.c .
9 *
10 * (C) Copyright 1995 1996 Linus Torvalds
11 *
12 * This file is subject to the terms and conditions of the GNU General Public
13 * License. See the file "COPYING" in the main directory of this archive
14 * for more details.
15 */
16#include <linux/vmalloc.h>
17#include <linux/ioport.h>
18#include <linux/module.h>
19#include <linux/mm.h>
20#include <linux/io.h>
21#include <linux/bootmem.h>
22#include <linux/proc_fs.h>
23#include <asm/page.h>
24#include <asm/pgalloc.h>
25#include <asm/addrspace.h>
26#include <asm/cacheflush.h>
27#include <asm/tlbflush.h>
28#include <asm/mmu.h>
29
30static void shmedia_mapioaddr(unsigned long, unsigned long);
31static unsigned long shmedia_ioremap(struct resource *, u32, int);
32
33/*
34 * Generic mapping function (not visible outside):
35 */
36
37/*
38 * Remap an arbitrary physical address space into the kernel virtual
39 * address space. Needed when the kernel wants to access high addresses
40 * directly.
41 *
42 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
43 * have to convert them into an offset in a page-aligned mapping, but the
44 * caller shouldn't need to know that small detail.
45 */
46void *__ioremap(unsigned long phys_addr, unsigned long size,
47 unsigned long flags)
48{
49 void * addr;
50 struct vm_struct * area;
51 unsigned long offset, last_addr;
52 pgprot_t pgprot;
53
54 /* Don't allow wraparound or zero size */
55 last_addr = phys_addr + size - 1;
56 if (!size || last_addr < phys_addr)
57 return NULL;
58
59 pgprot = __pgprot(_PAGE_PRESENT | _PAGE_READ |
60 _PAGE_WRITE | _PAGE_DIRTY |
61 _PAGE_ACCESSED | _PAGE_SHARED | flags);
62
63 /*
64 * Mappings have to be page-aligned
65 */
66 offset = phys_addr & ~PAGE_MASK;
67 phys_addr &= PAGE_MASK;
68 size = PAGE_ALIGN(last_addr + 1) - phys_addr;
69
70 /*
71 * Ok, go for it..
72 */
73 area = get_vm_area(size, VM_IOREMAP);
74 pr_debug("Get vm_area returns %p addr %p\n",area,area->addr);
75 if (!area)
76 return NULL;
77 area->phys_addr = phys_addr;
78 addr = area->addr;
79 if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
80 phys_addr, pgprot)) {
81 vunmap(addr);
82 return NULL;
83 }
84 return (void *) (offset + (char *)addr);
85}
86EXPORT_SYMBOL(__ioremap);
87
88void __iounmap(void *addr)
89{
90 struct vm_struct *area;
91
92 vfree((void *) (PAGE_MASK & (unsigned long) addr));
93 area = remove_vm_area((void *) (PAGE_MASK & (unsigned long) addr));
94 if (!area) {
95 printk(KERN_ERR "iounmap: bad address %p\n", addr);
96 return;
97 }
98
99 kfree(area);
100}
101EXPORT_SYMBOL(__iounmap);
102
103static struct resource shmedia_iomap = {
104 .name = "shmedia_iomap",
105 .start = IOBASE_VADDR + PAGE_SIZE,
106 .end = IOBASE_END - 1,
107};
108
109static void shmedia_mapioaddr(unsigned long pa, unsigned long va);
110static void shmedia_unmapioaddr(unsigned long vaddr);
111static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz);
112
113/*
114 * We have the same problem as the SPARC, so lets have the same comment:
115 * Our mini-allocator...
116 * Boy this is gross! We need it because we must map I/O for
117 * timers and interrupt controller before the kmalloc is available.
118 */
119
120#define XNMLN 15
121#define XNRES 10
122
123struct xresource {
124 struct resource xres; /* Must be first */
125 int xflag; /* 1 == used */
126 char xname[XNMLN+1];
127};
128
129static struct xresource xresv[XNRES];
130
131static struct xresource *xres_alloc(void)
132{
133 struct xresource *xrp;
134 int n;
135
136 xrp = xresv;
137 for (n = 0; n < XNRES; n++) {
138 if (xrp->xflag == 0) {
139 xrp->xflag = 1;
140 return xrp;
141 }
142 xrp++;
143 }
144 return NULL;
145}
146
147static void xres_free(struct xresource *xrp)
148{
149 xrp->xflag = 0;
150}
151
152static struct resource *shmedia_find_resource(struct resource *root,
153 unsigned long vaddr)
154{
155 struct resource *res;
156
157 for (res = root->child; res; res = res->sibling)
158 if (res->start <= vaddr && res->end >= vaddr)
159 return res;
160
161 return NULL;
162}
163
164static unsigned long shmedia_alloc_io(unsigned long phys, unsigned long size,
165 const char *name)
166{
167 static int printed_full = 0;
168 struct xresource *xres;
169 struct resource *res;
170 char *tack;
171 int tlen;
172
173 if (name == NULL) name = "???";
174
175 if ((xres = xres_alloc()) != 0) {
176 tack = xres->xname;
177 res = &xres->xres;
178 } else {
179 if (!printed_full) {
180 printk("%s: done with statics, switching to kmalloc\n",
181 __FUNCTION__);
182 printed_full = 1;
183 }
184 tlen = strlen(name);
185 tack = kmalloc(sizeof (struct resource) + tlen + 1, GFP_KERNEL);
186 if (!tack)
187 return -ENOMEM;
188 memset(tack, 0, sizeof(struct resource));
189 res = (struct resource *) tack;
190 tack += sizeof (struct resource);
191 }
192
193 strncpy(tack, name, XNMLN);
194 tack[XNMLN] = 0;
195 res->name = tack;
196
197 return shmedia_ioremap(res, phys, size);
198}
199
200static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz)
201{
202 unsigned long offset = ((unsigned long) pa) & (~PAGE_MASK);
203 unsigned long round_sz = (offset + sz + PAGE_SIZE-1) & PAGE_MASK;
204 unsigned long va;
205 unsigned int psz;
206
207 if (allocate_resource(&shmedia_iomap, res, round_sz,
208 shmedia_iomap.start, shmedia_iomap.end,
209 PAGE_SIZE, NULL, NULL) != 0) {
210 panic("alloc_io_res(%s): cannot occupy\n",
211 (res->name != NULL)? res->name: "???");
212 }
213
214 va = res->start;
215 pa &= PAGE_MASK;
216
217 psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE;
218
219 /* log at boot time ... */
220 printk("mapioaddr: %6s [%2d page%s] va 0x%08lx pa 0x%08x\n",
221 ((res->name != NULL) ? res->name : "???"),
222 psz, psz == 1 ? " " : "s", va, pa);
223
224 for (psz = res->end - res->start + 1; psz != 0; psz -= PAGE_SIZE) {
225 shmedia_mapioaddr(pa, va);
226 va += PAGE_SIZE;
227 pa += PAGE_SIZE;
228 }
229
230 res->start += offset;
231 res->end = res->start + sz - 1; /* not strictly necessary.. */
232
233 return res->start;
234}
235
236static void shmedia_free_io(struct resource *res)
237{
238 unsigned long len = res->end - res->start + 1;
239
240 BUG_ON((len & (PAGE_SIZE - 1)) != 0);
241
242 while (len) {
243 len -= PAGE_SIZE;
244 shmedia_unmapioaddr(res->start + len);
245 }
246
247 release_resource(res);
248}
249
250static __init_refok void *sh64_get_page(void)
251{
252 extern int after_bootmem;
253 void *page;
254
255 if (after_bootmem) {
256 page = (void *)get_zeroed_page(GFP_ATOMIC);
257 } else {
258 page = alloc_bootmem_pages(PAGE_SIZE);
259 }
260
261 if (!page || ((unsigned long)page & ~PAGE_MASK))
262 panic("sh64_get_page: Out of memory already?\n");
263
264 return page;
265}
266
267static void shmedia_mapioaddr(unsigned long pa, unsigned long va)
268{
269 pgd_t *pgdp;
270 pud_t *pudp;
271 pmd_t *pmdp;
272 pte_t *ptep, pte;
273 pgprot_t prot;
274 unsigned long flags = 1; /* 1 = CB0-1 device */
275
276 pr_debug("shmedia_mapiopage pa %08lx va %08lx\n", pa, va);
277
278 pgdp = pgd_offset_k(va);
279 if (pgd_none(*pgdp) || !pgd_present(*pgdp)) {
280 pudp = (pud_t *)sh64_get_page();
281 set_pgd(pgdp, __pgd((unsigned long)pudp | _KERNPG_TABLE));
282 }
283
284 pudp = pud_offset(pgdp, va);
285 if (pud_none(*pudp) || !pud_present(*pudp)) {
286 pmdp = (pmd_t *)sh64_get_page();
287 set_pud(pudp, __pud((unsigned long)pmdp | _KERNPG_TABLE));
288 }
289
290 pmdp = pmd_offset(pudp, va);
291 if (pmd_none(*pmdp) || !pmd_present(*pmdp) ) {
292 ptep = (pte_t *)sh64_get_page();
293 set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE));
294 }
295
296 prot = __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE |
297 _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SHARED | flags);
298
299 pte = pfn_pte(pa >> PAGE_SHIFT, prot);
300 ptep = pte_offset_kernel(pmdp, va);
301
302 if (!pte_none(*ptep) &&
303 pte_val(*ptep) != pte_val(pte))
304 pte_ERROR(*ptep);
305
306 set_pte(ptep, pte);
307
308 flush_tlb_kernel_range(va, PAGE_SIZE);
309}
310
311static void shmedia_unmapioaddr(unsigned long vaddr)
312{
313 pgd_t *pgdp;
314 pud_t *pudp;
315 pmd_t *pmdp;
316 pte_t *ptep;
317
318 pgdp = pgd_offset_k(vaddr);
319 if (pgd_none(*pgdp) || pgd_bad(*pgdp))
320 return;
321
322 pudp = pud_offset(pgdp, vaddr);
323 if (pud_none(*pudp) || pud_bad(*pudp))
324 return;
325
326 pmdp = pmd_offset(pudp, vaddr);
327 if (pmd_none(*pmdp) || pmd_bad(*pmdp))
328 return;
329
330 ptep = pte_offset_kernel(pmdp, vaddr);
331
332 if (pte_none(*ptep) || !pte_present(*ptep))
333 return;
334
335 clear_page((void *)ptep);
336 pte_clear(&init_mm, vaddr, ptep);
337}
338
339unsigned long onchip_remap(unsigned long phys, unsigned long size, const char *name)
340{
341 if (size < PAGE_SIZE)
342 size = PAGE_SIZE;
343
344 return shmedia_alloc_io(phys, size, name);
345}
346
347void onchip_unmap(unsigned long vaddr)
348{
349 struct resource *res;
350 unsigned int psz;
351
352 res = shmedia_find_resource(&shmedia_iomap, vaddr);
353 if (!res) {
354 printk(KERN_ERR "%s: Failed to free 0x%08lx\n",
355 __FUNCTION__, vaddr);
356 return;
357 }
358
359 psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE;
360
361 printk(KERN_DEBUG "unmapioaddr: %6s [%2d page%s] freed\n",
362 res->name, psz, psz == 1 ? " " : "s");
363
364 shmedia_free_io(res);
365
366 if ((char *)res >= (char *)xresv &&
367 (char *)res < (char *)&xresv[XNRES]) {
368 xres_free((struct xresource *)res);
369 } else {
370 kfree(res);
371 }
372}
373
374#ifdef CONFIG_PROC_FS
375static int
376ioremap_proc_info(char *buf, char **start, off_t fpos, int length, int *eof,
377 void *data)
378{
379 char *p = buf, *e = buf + length;
380 struct resource *r;
381 const char *nm;
382
383 for (r = ((struct resource *)data)->child; r != NULL; r = r->sibling) {
384 if (p + 32 >= e) /* Better than nothing */
385 break;
386 if ((nm = r->name) == 0) nm = "???";
387 p += sprintf(p, "%08lx-%08lx: %s\n",
388 (unsigned long)r->start,
389 (unsigned long)r->end, nm);
390 }
391
392 return p-buf;
393}
394#endif /* CONFIG_PROC_FS */
395
396static int __init register_proc_onchip(void)
397{
398#ifdef CONFIG_PROC_FS
399 create_proc_read_entry("io_map",0,0, ioremap_proc_info, &shmedia_iomap);
400#endif
401 return 0;
402}
403
404__initcall(register_proc_onchip);
diff --git a/arch/sh/mm/pg-nommu.c b/arch/sh/mm/pg-nommu.c
index d15221beaa16..677dd57f0877 100644
--- a/arch/sh/mm/pg-nommu.c
+++ b/arch/sh/mm/pg-nommu.c
@@ -14,12 +14,12 @@
14#include <linux/string.h> 14#include <linux/string.h>
15#include <asm/page.h> 15#include <asm/page.h>
16 16
17void copy_page_nommu(void *to, void *from) 17void copy_page(void *to, void *from)
18{ 18{
19 memcpy(to, from, PAGE_SIZE); 19 memcpy(to, from, PAGE_SIZE);
20} 20}
21 21
22void clear_page_nommu(void *to) 22void clear_page(void *to)
23{ 23{
24 memset(to, 0, PAGE_SIZE); 24 memset(to, 0, PAGE_SIZE);
25} 25}
diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c
index 1d45b82f0a63..ab81c602295f 100644
--- a/arch/sh/mm/pmb.c
+++ b/arch/sh/mm/pmb.c
@@ -27,6 +27,7 @@
27#include <asm/pgtable.h> 27#include <asm/pgtable.h>
28#include <asm/mmu.h> 28#include <asm/mmu.h>
29#include <asm/io.h> 29#include <asm/io.h>
30#include <asm/mmu_context.h>
30 31
31#define NR_PMB_ENTRIES 16 32#define NR_PMB_ENTRIES 16
32 33
@@ -162,18 +163,18 @@ repeat:
162 return 0; 163 return 0;
163} 164}
164 165
165int set_pmb_entry(struct pmb_entry *pmbe) 166int __uses_jump_to_uncached set_pmb_entry(struct pmb_entry *pmbe)
166{ 167{
167 int ret; 168 int ret;
168 169
169 jump_to_P2(); 170 jump_to_uncached();
170 ret = __set_pmb_entry(pmbe->vpn, pmbe->ppn, pmbe->flags, &pmbe->entry); 171 ret = __set_pmb_entry(pmbe->vpn, pmbe->ppn, pmbe->flags, &pmbe->entry);
171 back_to_P1(); 172 back_to_cached();
172 173
173 return ret; 174 return ret;
174} 175}
175 176
176void clear_pmb_entry(struct pmb_entry *pmbe) 177void __uses_jump_to_uncached clear_pmb_entry(struct pmb_entry *pmbe)
177{ 178{
178 unsigned int entry = pmbe->entry; 179 unsigned int entry = pmbe->entry;
179 unsigned long addr; 180 unsigned long addr;
@@ -187,7 +188,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe)
187 entry >= NR_PMB_ENTRIES)) 188 entry >= NR_PMB_ENTRIES))
188 return; 189 return;
189 190
190 jump_to_P2(); 191 jump_to_uncached();
191 192
192 /* Clear V-bit */ 193 /* Clear V-bit */
193 addr = mk_pmb_addr(entry); 194 addr = mk_pmb_addr(entry);
@@ -196,7 +197,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe)
196 addr = mk_pmb_data(entry); 197 addr = mk_pmb_data(entry);
197 ctrl_outl(ctrl_inl(addr) & ~PMB_V, addr); 198 ctrl_outl(ctrl_inl(addr) & ~PMB_V, addr);
198 199
199 back_to_P1(); 200 back_to_cached();
200 201
201 clear_bit(entry, &pmb_map); 202 clear_bit(entry, &pmb_map);
202} 203}
@@ -301,17 +302,17 @@ static void pmb_cache_ctor(struct kmem_cache *cachep, void *pmb)
301 pmbe->entry = PMB_NO_ENTRY; 302 pmbe->entry = PMB_NO_ENTRY;
302} 303}
303 304
304static int __init pmb_init(void) 305static int __uses_jump_to_uncached pmb_init(void)
305{ 306{
306 unsigned int nr_entries = ARRAY_SIZE(pmb_init_map); 307 unsigned int nr_entries = ARRAY_SIZE(pmb_init_map);
307 unsigned int entry; 308 unsigned int entry, i;
308 309
309 BUG_ON(unlikely(nr_entries >= NR_PMB_ENTRIES)); 310 BUG_ON(unlikely(nr_entries >= NR_PMB_ENTRIES));
310 311
311 pmb_cache = kmem_cache_create("pmb", sizeof(struct pmb_entry), 0, 312 pmb_cache = kmem_cache_create("pmb", sizeof(struct pmb_entry), 0,
312 SLAB_PANIC, pmb_cache_ctor); 313 SLAB_PANIC, pmb_cache_ctor);
313 314
314 jump_to_P2(); 315 jump_to_uncached();
315 316
316 /* 317 /*
317 * Ordering is important, P2 must be mapped in the PMB before we 318 * Ordering is important, P2 must be mapped in the PMB before we
@@ -329,7 +330,12 @@ static int __init pmb_init(void)
329 /* PMB.SE and UB[7] */ 330 /* PMB.SE and UB[7] */
330 ctrl_outl((1 << 31) | (1 << 7), PMB_PASCR); 331 ctrl_outl((1 << 31) | (1 << 7), PMB_PASCR);
331 332
332 back_to_P1(); 333 /* Flush out the TLB */
334 i = ctrl_inl(MMUCR);
335 i |= MMUCR_TI;
336 ctrl_outl(i, MMUCR);
337
338 back_to_cached();
333 339
334 return 0; 340 return 0;
335} 341}
diff --git a/arch/sh/mm/tlb-nommu.c b/arch/sh/mm/tlb-nommu.c
index 1ccca7c0532e..15111bc7ddd6 100644
--- a/arch/sh/mm/tlb-nommu.c
+++ b/arch/sh/mm/tlb-nommu.c
@@ -9,6 +9,7 @@
9 */ 9 */
10#include <linux/kernel.h> 10#include <linux/kernel.h>
11#include <linux/mm.h> 11#include <linux/mm.h>
12#include <asm/pgtable.h>
12 13
13/* 14/*
14 * Nothing too terribly exciting here .. 15 * Nothing too terribly exciting here ..
@@ -49,3 +50,12 @@ void update_mmu_cache(struct vm_area_struct * vma,
49{ 50{
50 BUG(); 51 BUG();
51} 52}
53
54void __init page_table_range_init(unsigned long start, unsigned long end,
55 pgd_t *pgd_base)
56{
57}
58
59void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
60{
61}
diff --git a/arch/sh/mm/tlb-sh4.c b/arch/sh/mm/tlb-sh4.c
index 2d1dd6044307..f0c7b7397fa6 100644
--- a/arch/sh/mm/tlb-sh4.c
+++ b/arch/sh/mm/tlb-sh4.c
@@ -79,7 +79,8 @@ void update_mmu_cache(struct vm_area_struct * vma,
79 local_irq_restore(flags); 79 local_irq_restore(flags);
80} 80}
81 81
82void local_flush_tlb_one(unsigned long asid, unsigned long page) 82void __uses_jump_to_uncached local_flush_tlb_one(unsigned long asid,
83 unsigned long page)
83{ 84{
84 unsigned long addr, data; 85 unsigned long addr, data;
85 86
@@ -91,7 +92,7 @@ void local_flush_tlb_one(unsigned long asid, unsigned long page)
91 */ 92 */
92 addr = MMU_UTLB_ADDRESS_ARRAY | MMU_PAGE_ASSOC_BIT; 93 addr = MMU_UTLB_ADDRESS_ARRAY | MMU_PAGE_ASSOC_BIT;
93 data = page | asid; /* VALID bit is off */ 94 data = page | asid; /* VALID bit is off */
94 jump_to_P2(); 95 jump_to_uncached();
95 ctrl_outl(data, addr); 96 ctrl_outl(data, addr);
96 back_to_P1(); 97 back_to_cached();
97} 98}
diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c
new file mode 100644
index 000000000000..f34274a1ded3
--- /dev/null
+++ b/arch/sh/mm/tlb-sh5.c
@@ -0,0 +1,164 @@
1/*
2 * arch/sh/mm/tlb-sh5.c
3 *
4 * Copyright (C) 2003 Paul Mundt <lethal@linux-sh.org>
5 * Copyright (C) 2003 Richard Curnow <richard.curnow@superh.com>
6 *
7 * This file is subject to the terms and conditions of the GNU General Public
8 * License. See the file "COPYING" in the main directory of this archive
9 * for more details.
10 */
11#include <linux/mm.h>
12#include <linux/init.h>
13#include <asm/page.h>
14#include <asm/tlb.h>
15#include <asm/mmu_context.h>
16
17/**
18 * sh64_tlb_init
19 *
20 * Perform initial setup for the DTLB and ITLB.
21 */
22int __init sh64_tlb_init(void)
23{
24 /* Assign some sane DTLB defaults */
25 cpu_data->dtlb.entries = 64;
26 cpu_data->dtlb.step = 0x10;
27
28 cpu_data->dtlb.first = DTLB_FIXED | cpu_data->dtlb.step;
29 cpu_data->dtlb.next = cpu_data->dtlb.first;
30
31 cpu_data->dtlb.last = DTLB_FIXED |
32 ((cpu_data->dtlb.entries - 1) *
33 cpu_data->dtlb.step);
34
35 /* And again for the ITLB */
36 cpu_data->itlb.entries = 64;
37 cpu_data->itlb.step = 0x10;
38
39 cpu_data->itlb.first = ITLB_FIXED | cpu_data->itlb.step;
40 cpu_data->itlb.next = cpu_data->itlb.first;
41 cpu_data->itlb.last = ITLB_FIXED |
42 ((cpu_data->itlb.entries - 1) *
43 cpu_data->itlb.step);
44
45 return 0;
46}
47
48/**
49 * sh64_next_free_dtlb_entry
50 *
51 * Find the next available DTLB entry
52 */
53unsigned long long sh64_next_free_dtlb_entry(void)
54{
55 return cpu_data->dtlb.next;
56}
57
58/**
59 * sh64_get_wired_dtlb_entry
60 *
61 * Allocate a wired (locked-in) entry in the DTLB
62 */
63unsigned long long sh64_get_wired_dtlb_entry(void)
64{
65 unsigned long long entry = sh64_next_free_dtlb_entry();
66
67 cpu_data->dtlb.first += cpu_data->dtlb.step;
68 cpu_data->dtlb.next += cpu_data->dtlb.step;
69
70 return entry;
71}
72
73/**
74 * sh64_put_wired_dtlb_entry
75 *
76 * @entry: Address of TLB slot.
77 *
78 * Free a wired (locked-in) entry in the DTLB.
79 *
80 * Works like a stack, last one to allocate must be first one to free.
81 */
82int sh64_put_wired_dtlb_entry(unsigned long long entry)
83{
84 __flush_tlb_slot(entry);
85
86 /*
87 * We don't do any particularly useful tracking of wired entries,
88 * so this approach works like a stack .. last one to be allocated
89 * has to be the first one to be freed.
90 *
91 * We could potentially load wired entries into a list and work on
92 * rebalancing the list periodically (which also entails moving the
93 * contents of a TLB entry) .. though I have a feeling that this is
94 * more trouble than it's worth.
95 */
96
97 /*
98 * Entry must be valid .. we don't want any ITLB addresses!
99 */
100 if (entry <= DTLB_FIXED)
101 return -EINVAL;
102
103 /*
104 * Next, check if we're within range to be freed. (ie, must be the
105 * entry beneath the first 'free' entry!
106 */
107 if (entry < (cpu_data->dtlb.first - cpu_data->dtlb.step))
108 return -EINVAL;
109
110 /* If we are, then bring this entry back into the list */
111 cpu_data->dtlb.first -= cpu_data->dtlb.step;
112 cpu_data->dtlb.next = entry;
113
114 return 0;
115}
116
117/**
118 * sh64_setup_tlb_slot
119 *
120 * @config_addr: Address of TLB slot.
121 * @eaddr: Virtual address.
122 * @asid: Address Space Identifier.
123 * @paddr: Physical address.
124 *
125 * Load up a virtual<->physical translation for @eaddr<->@paddr in the
126 * pre-allocated TLB slot @config_addr (see sh64_get_wired_dtlb_entry).
127 */
128inline void sh64_setup_tlb_slot(unsigned long long config_addr,
129 unsigned long eaddr,
130 unsigned long asid,
131 unsigned long paddr)
132{
133 unsigned long long pteh, ptel;
134
135 /* Sign extension */
136#if (NEFF == 32)
137 pteh = (unsigned long long)(signed long long)(signed long) eaddr;
138#else
139#error "Can't sign extend more than 32 bits yet"
140#endif
141 pteh &= PAGE_MASK;
142 pteh |= (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
143#if (NEFF == 32)
144 ptel = (unsigned long long)(signed long long)(signed long) paddr;
145#else
146#error "Can't sign extend more than 32 bits yet"
147#endif
148 ptel &= PAGE_MASK;
149 ptel |= (_PAGE_CACHABLE | _PAGE_READ | _PAGE_WRITE);
150
151 asm volatile("putcfg %0, 1, %1\n\t"
152 "putcfg %0, 0, %2\n"
153 : : "r" (config_addr), "r" (ptel), "r" (pteh));
154}
155
156/**
157 * sh64_teardown_tlb_slot
158 *
159 * @config_addr: Address of TLB slot.
160 *
161 * Teardown any existing mapping in the TLB slot @config_addr.
162 */
163inline void sh64_teardown_tlb_slot(unsigned long long config_addr)
164 __attribute__ ((alias("__flush_tlb_slot")));
diff --git a/arch/sh/mm/tlb-flush.c b/arch/sh/mm/tlbflush_32.c
index 6f45c1f8a7fe..6f45c1f8a7fe 100644
--- a/arch/sh/mm/tlb-flush.c
+++ b/arch/sh/mm/tlbflush_32.c
diff --git a/arch/sh/mm/tlbflush_64.c b/arch/sh/mm/tlbflush_64.c
new file mode 100644
index 000000000000..2a98c9ec88ff
--- /dev/null
+++ b/arch/sh/mm/tlbflush_64.c
@@ -0,0 +1,475 @@
1/*
2 * arch/sh/mm/tlb-flush_64.c
3 *
4 * Copyright (C) 2000, 2001 Paolo Alberelli
5 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
6 * Copyright (C) 2003 Paul Mundt
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 */
12#include <linux/signal.h>
13#include <linux/rwsem.h>
14#include <linux/sched.h>
15#include <linux/kernel.h>
16#include <linux/errno.h>
17#include <linux/string.h>
18#include <linux/types.h>
19#include <linux/ptrace.h>
20#include <linux/mman.h>
21#include <linux/mm.h>
22#include <linux/smp.h>
23#include <linux/interrupt.h>
24#include <asm/system.h>
25#include <asm/io.h>
26#include <asm/tlb.h>
27#include <asm/uaccess.h>
28#include <asm/pgalloc.h>
29#include <asm/mmu_context.h>
30
31extern void die(const char *,struct pt_regs *,long);
32
33#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
34#define PPROT(flag) PFLAG(pgprot_val(prot),flag)
35
36static inline void print_prots(pgprot_t prot)
37{
38 printk("prot is 0x%08lx\n",pgprot_val(prot));
39
40 printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
41 PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
42}
43
44static inline void print_vma(struct vm_area_struct *vma)
45{
46 printk("vma start 0x%08lx\n", vma->vm_start);
47 printk("vma end 0x%08lx\n", vma->vm_end);
48
49 print_prots(vma->vm_page_prot);
50 printk("vm_flags 0x%08lx\n", vma->vm_flags);
51}
52
53static inline void print_task(struct task_struct *tsk)
54{
55 printk("Task pid %d\n", task_pid_nr(tsk));
56}
57
58static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
59{
60 pgd_t *dir;
61 pud_t *pud;
62 pmd_t *pmd;
63 pte_t *pte;
64 pte_t entry;
65
66 dir = pgd_offset(mm, address);
67 if (pgd_none(*dir))
68 return NULL;
69
70 pud = pud_offset(dir, address);
71 if (pud_none(*pud))
72 return NULL;
73
74 pmd = pmd_offset(pud, address);
75 if (pmd_none(*pmd))
76 return NULL;
77
78 pte = pte_offset_kernel(pmd, address);
79 entry = *pte;
80 if (pte_none(entry) || !pte_present(entry))
81 return NULL;
82
83 return pte;
84}
85
86/*
87 * This routine handles page faults. It determines the address,
88 * and the problem, and then passes it off to one of the appropriate
89 * routines.
90 */
91asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
92 unsigned long textaccess, unsigned long address)
93{
94 struct task_struct *tsk;
95 struct mm_struct *mm;
96 struct vm_area_struct * vma;
97 const struct exception_table_entry *fixup;
98 pte_t *pte;
99 int fault;
100
101 /* SIM
102 * Note this is now called with interrupts still disabled
103 * This is to cope with being called for a missing IO port
104 * address with interrupts disabled. This should be fixed as
105 * soon as we have a better 'fast path' miss handler.
106 *
107 * Plus take care how you try and debug this stuff.
108 * For example, writing debug data to a port which you
109 * have just faulted on is not going to work.
110 */
111
112 tsk = current;
113 mm = tsk->mm;
114
115 /* Not an IO address, so reenable interrupts */
116 local_irq_enable();
117
118 /*
119 * If we're in an interrupt or have no user
120 * context, we must not take the fault..
121 */
122 if (in_atomic() || !mm)
123 goto no_context;
124
125 /* TLB misses upon some cache flushes get done under cli() */
126 down_read(&mm->mmap_sem);
127
128 vma = find_vma(mm, address);
129
130 if (!vma) {
131#ifdef DEBUG_FAULT
132 print_task(tsk);
133 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
134 __FUNCTION__,__LINE__,
135 address,regs->pc,textaccess,writeaccess);
136 show_regs(regs);
137#endif
138 goto bad_area;
139 }
140 if (vma->vm_start <= address) {
141 goto good_area;
142 }
143
144 if (!(vma->vm_flags & VM_GROWSDOWN)) {
145#ifdef DEBUG_FAULT
146 print_task(tsk);
147 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
148 __FUNCTION__,__LINE__,
149 address,regs->pc,textaccess,writeaccess);
150 show_regs(regs);
151
152 print_vma(vma);
153#endif
154 goto bad_area;
155 }
156 if (expand_stack(vma, address)) {
157#ifdef DEBUG_FAULT
158 print_task(tsk);
159 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
160 __FUNCTION__,__LINE__,
161 address,regs->pc,textaccess,writeaccess);
162 show_regs(regs);
163#endif
164 goto bad_area;
165 }
166/*
167 * Ok, we have a good vm_area for this memory access, so
168 * we can handle it..
169 */
170good_area:
171 if (textaccess) {
172 if (!(vma->vm_flags & VM_EXEC))
173 goto bad_area;
174 } else {
175 if (writeaccess) {
176 if (!(vma->vm_flags & VM_WRITE))
177 goto bad_area;
178 } else {
179 if (!(vma->vm_flags & VM_READ))
180 goto bad_area;
181 }
182 }
183
184 /*
185 * If for any reason at all we couldn't handle the fault,
186 * make sure we exit gracefully rather than endlessly redo
187 * the fault.
188 */
189survive:
190 fault = handle_mm_fault(mm, vma, address, writeaccess);
191 if (unlikely(fault & VM_FAULT_ERROR)) {
192 if (fault & VM_FAULT_OOM)
193 goto out_of_memory;
194 else if (fault & VM_FAULT_SIGBUS)
195 goto do_sigbus;
196 BUG();
197 }
198 if (fault & VM_FAULT_MAJOR)
199 tsk->maj_flt++;
200 else
201 tsk->min_flt++;
202
203 /* If we get here, the page fault has been handled. Do the TLB refill
204 now from the newly-setup PTE, to avoid having to fault again right
205 away on the same instruction. */
206 pte = lookup_pte (mm, address);
207 if (!pte) {
208 /* From empirical evidence, we can get here, due to
209 !pte_present(pte). (e.g. if a swap-in occurs, and the page
210 is swapped back out again before the process that wanted it
211 gets rescheduled?) */
212 goto no_pte;
213 }
214
215 __do_tlb_refill(address, textaccess, pte);
216
217no_pte:
218
219 up_read(&mm->mmap_sem);
220 return;
221
222/*
223 * Something tried to access memory that isn't in our memory map..
224 * Fix it, but check if it's kernel or user first..
225 */
226bad_area:
227#ifdef DEBUG_FAULT
228 printk("fault:bad area\n");
229#endif
230 up_read(&mm->mmap_sem);
231
232 if (user_mode(regs)) {
233 static int count=0;
234 siginfo_t info;
235 if (count < 4) {
236 /* This is really to help debug faults when starting
237 * usermode, so only need a few */
238 count++;
239 printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
240 address, task_pid_nr(current), current->comm,
241 (unsigned long) regs->pc);
242#if 0
243 show_regs(regs);
244#endif
245 }
246 if (is_global_init(tsk)) {
247 panic("INIT had user mode bad_area\n");
248 }
249 tsk->thread.address = address;
250 tsk->thread.error_code = writeaccess;
251 info.si_signo = SIGSEGV;
252 info.si_errno = 0;
253 info.si_addr = (void *) address;
254 force_sig_info(SIGSEGV, &info, tsk);
255 return;
256 }
257
258no_context:
259#ifdef DEBUG_FAULT
260 printk("fault:No context\n");
261#endif
262 /* Are we prepared to handle this kernel fault? */
263 fixup = search_exception_tables(regs->pc);
264 if (fixup) {
265 regs->pc = fixup->fixup;
266 return;
267 }
268
269/*
270 * Oops. The kernel tried to access some bad page. We'll have to
271 * terminate things with extreme prejudice.
272 *
273 */
274 if (address < PAGE_SIZE)
275 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
276 else
277 printk(KERN_ALERT "Unable to handle kernel paging request");
278 printk(" at virtual address %08lx\n", address);
279 printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
280 die("Oops", regs, writeaccess);
281 do_exit(SIGKILL);
282
283/*
284 * We ran out of memory, or some other thing happened to us that made
285 * us unable to handle the page fault gracefully.
286 */
287out_of_memory:
288 if (is_global_init(current)) {
289 panic("INIT out of memory\n");
290 yield();
291 goto survive;
292 }
293 printk("fault:Out of memory\n");
294 up_read(&mm->mmap_sem);
295 if (is_global_init(current)) {
296 yield();
297 down_read(&mm->mmap_sem);
298 goto survive;
299 }
300 printk("VM: killing process %s\n", tsk->comm);
301 if (user_mode(regs))
302 do_group_exit(SIGKILL);
303 goto no_context;
304
305do_sigbus:
306 printk("fault:Do sigbus\n");
307 up_read(&mm->mmap_sem);
308
309 /*
310 * Send a sigbus, regardless of whether we were in kernel
311 * or user mode.
312 */
313 tsk->thread.address = address;
314 tsk->thread.error_code = writeaccess;
315 tsk->thread.trap_no = 14;
316 force_sig(SIGBUS, tsk);
317
318 /* Kernel mode? Handle exceptions or die */
319 if (!user_mode(regs))
320 goto no_context;
321}
322
323void update_mmu_cache(struct vm_area_struct * vma,
324 unsigned long address, pte_t pte)
325{
326 /*
327 * This appears to get called once for every pte entry that gets
328 * established => I don't think it's efficient to try refilling the
329 * TLBs with the pages - some may not get accessed even. Also, for
330 * executable pages, it is impossible to determine reliably here which
331 * TLB they should be mapped into (or both even).
332 *
333 * So, just do nothing here and handle faults on demand. In the
334 * TLBMISS handling case, the refill is now done anyway after the pte
335 * has been fixed up, so that deals with most useful cases.
336 */
337}
338
339void local_flush_tlb_one(unsigned long asid, unsigned long page)
340{
341 unsigned long long match, pteh=0, lpage;
342 unsigned long tlb;
343
344 /*
345 * Sign-extend based on neff.
346 */
347 lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page;
348 match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
349 match |= lpage;
350
351 for_each_itlb_entry(tlb) {
352 asm volatile ("getcfg %1, 0, %0"
353 : "=r" (pteh)
354 : "r" (tlb) );
355
356 if (pteh == match) {
357 __flush_tlb_slot(tlb);
358 break;
359 }
360 }
361
362 for_each_dtlb_entry(tlb) {
363 asm volatile ("getcfg %1, 0, %0"
364 : "=r" (pteh)
365 : "r" (tlb) );
366
367 if (pteh == match) {
368 __flush_tlb_slot(tlb);
369 break;
370 }
371
372 }
373}
374
375void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
376{
377 unsigned long flags;
378
379 if (vma->vm_mm) {
380 page &= PAGE_MASK;
381 local_irq_save(flags);
382 local_flush_tlb_one(get_asid(), page);
383 local_irq_restore(flags);
384 }
385}
386
387void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
388 unsigned long end)
389{
390 unsigned long flags;
391 unsigned long long match, pteh=0, pteh_epn, pteh_low;
392 unsigned long tlb;
393 unsigned int cpu = smp_processor_id();
394 struct mm_struct *mm;
395
396 mm = vma->vm_mm;
397 if (cpu_context(cpu, mm) == NO_CONTEXT)
398 return;
399
400 local_irq_save(flags);
401
402 start &= PAGE_MASK;
403 end &= PAGE_MASK;
404
405 match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
406
407 /* Flush ITLB */
408 for_each_itlb_entry(tlb) {
409 asm volatile ("getcfg %1, 0, %0"
410 : "=r" (pteh)
411 : "r" (tlb) );
412
413 pteh_epn = pteh & PAGE_MASK;
414 pteh_low = pteh & ~PAGE_MASK;
415
416 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
417 __flush_tlb_slot(tlb);
418 }
419
420 /* Flush DTLB */
421 for_each_dtlb_entry(tlb) {
422 asm volatile ("getcfg %1, 0, %0"
423 : "=r" (pteh)
424 : "r" (tlb) );
425
426 pteh_epn = pteh & PAGE_MASK;
427 pteh_low = pteh & ~PAGE_MASK;
428
429 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
430 __flush_tlb_slot(tlb);
431 }
432
433 local_irq_restore(flags);
434}
435
436void local_flush_tlb_mm(struct mm_struct *mm)
437{
438 unsigned long flags;
439 unsigned int cpu = smp_processor_id();
440
441 if (cpu_context(cpu, mm) == NO_CONTEXT)
442 return;
443
444 local_irq_save(flags);
445
446 cpu_context(cpu, mm) = NO_CONTEXT;
447 if (mm == current->mm)
448 activate_context(mm, cpu);
449
450 local_irq_restore(flags);
451}
452
453void local_flush_tlb_all(void)
454{
455 /* Invalidate all, including shared pages, excluding fixed TLBs */
456 unsigned long flags, tlb;
457
458 local_irq_save(flags);
459
460 /* Flush each ITLB entry */
461 for_each_itlb_entry(tlb)
462 __flush_tlb_slot(tlb);
463
464 /* Flush each DTLB entry */
465 for_each_dtlb_entry(tlb)
466 __flush_tlb_slot(tlb);
467
468 local_irq_restore(flags);
469}
470
471void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
472{
473 /* FIXME: Optimize this later.. */
474 flush_tlb_all();
475}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-21 13:58:05 -0500