aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/mm')
-rw-r--r--arch/mips/mm/Makefile44
-rw-r--r--arch/mips/mm/c-r3k.c349
-rw-r--r--arch/mips/mm/c-r4k.c1260
-rw-r--r--arch/mips/mm/c-sb1.c558
-rw-r--r--arch/mips/mm/c-tx39.c493
-rw-r--r--arch/mips/mm/cache.c157
-rw-r--r--arch/mips/mm/cerr-sb1.c543
-rw-r--r--arch/mips/mm/cex-gen.S42
-rw-r--r--arch/mips/mm/cex-sb1.S170
-rw-r--r--arch/mips/mm/dma-coherent.c255
-rw-r--r--arch/mips/mm/dma-ip27.c257
-rw-r--r--arch/mips/mm/dma-ip32.c382
-rw-r--r--arch/mips/mm/dma-noncoherent.c400
-rw-r--r--arch/mips/mm/extable.c21
-rw-r--r--arch/mips/mm/fault.c236
-rw-r--r--arch/mips/mm/highmem.c103
-rw-r--r--arch/mips/mm/init.c304
-rw-r--r--arch/mips/mm/ioremap.c202
-rw-r--r--arch/mips/mm/pg-r4k.c489
-rw-r--r--arch/mips/mm/pg-sb1.c287
-rw-r--r--arch/mips/mm/pgtable-32.c97
-rw-r--r--arch/mips/mm/pgtable-64.c58
-rw-r--r--arch/mips/mm/pgtable.c36
-rw-r--r--arch/mips/mm/sc-ip22.c177
-rw-r--r--arch/mips/mm/sc-r5k.c108
-rw-r--r--arch/mips/mm/sc-rm7k.c193
-rw-r--r--arch/mips/mm/tlb-andes.c257
-rw-r--r--arch/mips/mm/tlb-r3k.c289
-rw-r--r--arch/mips/mm/tlb-r4k.c419
-rw-r--r--arch/mips/mm/tlb-r8k.c250
-rw-r--r--arch/mips/mm/tlb-sb1.c376
-rw-r--r--arch/mips/mm/tlbex-fault.S28
-rw-r--r--arch/mips/mm/tlbex.c1815
33 files changed, 10655 insertions, 0 deletions
diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile
new file mode 100644
index 000000000000..f61e038b4440
--- /dev/null
+++ b/arch/mips/mm/Makefile
@@ -0,0 +1,44 @@
1#
2# Makefile for the Linux/MIPS-specific parts of the memory manager.
3#
4
5obj-y += cache.o extable.o fault.o init.o pgtable.o \
6 tlbex.o tlbex-fault.o
7
8obj-$(CONFIG_MIPS32) += ioremap.o pgtable-32.o
9obj-$(CONFIG_MIPS64) += pgtable-64.o
10obj-$(CONFIG_HIGHMEM) += highmem.o
11
12obj-$(CONFIG_CPU_MIPS32) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
13obj-$(CONFIG_CPU_MIPS64) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
14obj-$(CONFIG_CPU_NEVADA) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
15obj-$(CONFIG_CPU_R10000) += c-r4k.o cex-gen.o pg-r4k.o tlb-andes.o
16obj-$(CONFIG_CPU_R3000) += c-r3k.o tlb-r3k.o pg-r4k.o
17obj-$(CONFIG_CPU_R4300) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
18obj-$(CONFIG_CPU_R4X00) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
19obj-$(CONFIG_CPU_R5000) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
20obj-$(CONFIG_CPU_R5432) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
21obj-$(CONFIG_CPU_R8000) += c-r4k.o cex-gen.o pg-r4k.o tlb-r8k.o
22obj-$(CONFIG_CPU_RM7000) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
23obj-$(CONFIG_CPU_RM9000) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
24obj-$(CONFIG_CPU_SB1) += c-sb1.o cerr-sb1.o cex-sb1.o pg-sb1.o \
25 tlb-sb1.o
26obj-$(CONFIG_CPU_TX39XX) += c-tx39.o pg-r4k.o tlb-r3k.o
27obj-$(CONFIG_CPU_TX49XX) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
28obj-$(CONFIG_CPU_VR41XX) += c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
29
30obj-$(CONFIG_IP22_CPU_SCACHE) += sc-ip22.o
31obj-$(CONFIG_R5000_CPU_SCACHE) += sc-r5k.o
32obj-$(CONFIG_RM7000_CPU_SCACHE) += sc-rm7k.o
33
34#
35# Choose one DMA coherency model
36#
37ifndef CONFIG_OWN_DMA
38obj-$(CONFIG_DMA_COHERENT) += dma-coherent.o
39obj-$(CONFIG_DMA_NONCOHERENT) += dma-noncoherent.o
40endif
41obj-$(CONFIG_DMA_IP27) += dma-ip27.o
42obj-$(CONFIG_DMA_IP32) += dma-ip32.o
43
44EXTRA_AFLAGS := $(CFLAGS)
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
new file mode 100644
index 000000000000..c659f99eb39a
--- /dev/null
+++ b/arch/mips/mm/c-r3k.c
@@ -0,0 +1,349 @@
1/*
2 * r2300.c: R2000 and R3000 specific mmu/cache code.
3 *
4 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
5 *
6 * with a lot of changes to make this thing work for R3000s
7 * Tx39XX R4k style caches added. HK
8 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
9 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
10 * Copyright (C) 2001, 2004 Maciej W. Rozycki
11 */
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/mm.h>
16
17#include <asm/page.h>
18#include <asm/pgtable.h>
19#include <asm/mmu_context.h>
20#include <asm/system.h>
21#include <asm/isadep.h>
22#include <asm/io.h>
23#include <asm/bootinfo.h>
24#include <asm/cpu.h>
25
26static unsigned long icache_size, dcache_size; /* Size in bytes */
27static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */
28
29#undef DEBUG_CACHE
30
31unsigned long __init r3k_cache_size(unsigned long ca_flags)
32{
33 unsigned long flags, status, dummy, size;
34 volatile unsigned long *p;
35
36 p = (volatile unsigned long *) KSEG0;
37
38 flags = read_c0_status();
39
40 /* isolate cache space */
41 write_c0_status((ca_flags|flags)&~ST0_IEC);
42
43 *p = 0xa5a55a5a;
44 dummy = *p;
45 status = read_c0_status();
46
47 if (dummy != 0xa5a55a5a || (status & ST0_CM)) {
48 size = 0;
49 } else {
50 for (size = 128; size <= 0x40000; size <<= 1)
51 *(p + size) = 0;
52 *p = -1;
53 for (size = 128;
54 (size <= 0x40000) && (*(p + size) == 0);
55 size <<= 1)
56 ;
57 if (size > 0x40000)
58 size = 0;
59 }
60
61 write_c0_status(flags);
62
63 return size * sizeof(*p);
64}
65
66unsigned long __init r3k_cache_lsize(unsigned long ca_flags)
67{
68 unsigned long flags, status, lsize, i;
69 volatile unsigned long *p;
70
71 p = (volatile unsigned long *) KSEG0;
72
73 flags = read_c0_status();
74
75 /* isolate cache space */
76 write_c0_status((ca_flags|flags)&~ST0_IEC);
77
78 for (i = 0; i < 128; i++)
79 *(p + i) = 0;
80 *(volatile unsigned char *)p = 0;
81 for (lsize = 1; lsize < 128; lsize <<= 1) {
82 *(p + lsize);
83 status = read_c0_status();
84 if (!(status & ST0_CM))
85 break;
86 }
87 for (i = 0; i < 128; i += lsize)
88 *(volatile unsigned char *)(p + i) = 0;
89
90 write_c0_status(flags);
91
92 return lsize * sizeof(*p);
93}
94
95static void __init r3k_probe_cache(void)
96{
97 dcache_size = r3k_cache_size(ST0_ISC);
98 if (dcache_size)
99 dcache_lsize = r3k_cache_lsize(ST0_ISC);
100
101 icache_size = r3k_cache_size(ST0_ISC|ST0_SWC);
102 if (icache_size)
103 icache_lsize = r3k_cache_lsize(ST0_ISC|ST0_SWC);
104}
105
106static void r3k_flush_icache_range(unsigned long start, unsigned long end)
107{
108 unsigned long size, i, flags;
109 volatile unsigned char *p;
110
111 size = end - start;
112 if (size > icache_size || KSEGX(start) != KSEG0) {
113 start = KSEG0;
114 size = icache_size;
115 }
116 p = (char *)start;
117
118 flags = read_c0_status();
119
120 /* isolate cache space */
121 write_c0_status((ST0_ISC|ST0_SWC|flags)&~ST0_IEC);
122
123 for (i = 0; i < size; i += 0x080) {
124 asm ( "sb\t$0, 0x000(%0)\n\t"
125 "sb\t$0, 0x004(%0)\n\t"
126 "sb\t$0, 0x008(%0)\n\t"
127 "sb\t$0, 0x00c(%0)\n\t"
128 "sb\t$0, 0x010(%0)\n\t"
129 "sb\t$0, 0x014(%0)\n\t"
130 "sb\t$0, 0x018(%0)\n\t"
131 "sb\t$0, 0x01c(%0)\n\t"
132 "sb\t$0, 0x020(%0)\n\t"
133 "sb\t$0, 0x024(%0)\n\t"
134 "sb\t$0, 0x028(%0)\n\t"
135 "sb\t$0, 0x02c(%0)\n\t"
136 "sb\t$0, 0x030(%0)\n\t"
137 "sb\t$0, 0x034(%0)\n\t"
138 "sb\t$0, 0x038(%0)\n\t"
139 "sb\t$0, 0x03c(%0)\n\t"
140 "sb\t$0, 0x040(%0)\n\t"
141 "sb\t$0, 0x044(%0)\n\t"
142 "sb\t$0, 0x048(%0)\n\t"
143 "sb\t$0, 0x04c(%0)\n\t"
144 "sb\t$0, 0x050(%0)\n\t"
145 "sb\t$0, 0x054(%0)\n\t"
146 "sb\t$0, 0x058(%0)\n\t"
147 "sb\t$0, 0x05c(%0)\n\t"
148 "sb\t$0, 0x060(%0)\n\t"
149 "sb\t$0, 0x064(%0)\n\t"
150 "sb\t$0, 0x068(%0)\n\t"
151 "sb\t$0, 0x06c(%0)\n\t"
152 "sb\t$0, 0x070(%0)\n\t"
153 "sb\t$0, 0x074(%0)\n\t"
154 "sb\t$0, 0x078(%0)\n\t"
155 "sb\t$0, 0x07c(%0)\n\t"
156 : : "r" (p) );
157 p += 0x080;
158 }
159
160 write_c0_status(flags);
161}
162
163static void r3k_flush_dcache_range(unsigned long start, unsigned long end)
164{
165 unsigned long size, i, flags;
166 volatile unsigned char *p;
167
168 size = end - start;
169 if (size > dcache_size || KSEGX(start) != KSEG0) {
170 start = KSEG0;
171 size = dcache_size;
172 }
173 p = (char *)start;
174
175 flags = read_c0_status();
176
177 /* isolate cache space */
178 write_c0_status((ST0_ISC|flags)&~ST0_IEC);
179
180 for (i = 0; i < size; i += 0x080) {
181 asm ( "sb\t$0, 0x000(%0)\n\t"
182 "sb\t$0, 0x004(%0)\n\t"
183 "sb\t$0, 0x008(%0)\n\t"
184 "sb\t$0, 0x00c(%0)\n\t"
185 "sb\t$0, 0x010(%0)\n\t"
186 "sb\t$0, 0x014(%0)\n\t"
187 "sb\t$0, 0x018(%0)\n\t"
188 "sb\t$0, 0x01c(%0)\n\t"
189 "sb\t$0, 0x020(%0)\n\t"
190 "sb\t$0, 0x024(%0)\n\t"
191 "sb\t$0, 0x028(%0)\n\t"
192 "sb\t$0, 0x02c(%0)\n\t"
193 "sb\t$0, 0x030(%0)\n\t"
194 "sb\t$0, 0x034(%0)\n\t"
195 "sb\t$0, 0x038(%0)\n\t"
196 "sb\t$0, 0x03c(%0)\n\t"
197 "sb\t$0, 0x040(%0)\n\t"
198 "sb\t$0, 0x044(%0)\n\t"
199 "sb\t$0, 0x048(%0)\n\t"
200 "sb\t$0, 0x04c(%0)\n\t"
201 "sb\t$0, 0x050(%0)\n\t"
202 "sb\t$0, 0x054(%0)\n\t"
203 "sb\t$0, 0x058(%0)\n\t"
204 "sb\t$0, 0x05c(%0)\n\t"
205 "sb\t$0, 0x060(%0)\n\t"
206 "sb\t$0, 0x064(%0)\n\t"
207 "sb\t$0, 0x068(%0)\n\t"
208 "sb\t$0, 0x06c(%0)\n\t"
209 "sb\t$0, 0x070(%0)\n\t"
210 "sb\t$0, 0x074(%0)\n\t"
211 "sb\t$0, 0x078(%0)\n\t"
212 "sb\t$0, 0x07c(%0)\n\t"
213 : : "r" (p) );
214 p += 0x080;
215 }
216
217 write_c0_status(flags);
218}
219
220static inline unsigned long get_phys_page (unsigned long addr,
221 struct mm_struct *mm)
222{
223 pgd_t *pgd;
224 pmd_t *pmd;
225 pte_t *pte;
226 unsigned long physpage;
227
228 pgd = pgd_offset(mm, addr);
229 pmd = pmd_offset(pgd, addr);
230 pte = pte_offset(pmd, addr);
231
232 if ((physpage = pte_val(*pte)) & _PAGE_VALID)
233 return KSEG0ADDR(physpage & PAGE_MASK);
234
235 return 0;
236}
237
238static inline void r3k_flush_cache_all(void)
239{
240}
241
242static inline void r3k___flush_cache_all(void)
243{
244 r3k_flush_dcache_range(KSEG0, KSEG0 + dcache_size);
245 r3k_flush_icache_range(KSEG0, KSEG0 + icache_size);
246}
247
248static void r3k_flush_cache_mm(struct mm_struct *mm)
249{
250}
251
252static void r3k_flush_cache_range(struct vm_area_struct *vma,
253 unsigned long start, unsigned long end)
254{
255}
256
257static void r3k_flush_cache_page(struct vm_area_struct *vma, unsigned long page, unsigned long pfn)
258{
259}
260
261static void r3k_flush_data_cache_page(unsigned long addr)
262{
263}
264
265static void r3k_flush_icache_page(struct vm_area_struct *vma, struct page *page)
266{
267 struct mm_struct *mm = vma->vm_mm;
268 unsigned long physpage;
269
270 if (cpu_context(smp_processor_id(), mm) == 0)
271 return;
272
273 if (!(vma->vm_flags & VM_EXEC))
274 return;
275
276#ifdef DEBUG_CACHE
277 printk("cpage[%d,%08lx]", cpu_context(smp_processor_id(), mm), page);
278#endif
279
280 physpage = (unsigned long) page_address(page);
281 if (physpage)
282 r3k_flush_icache_range(physpage, physpage + PAGE_SIZE);
283}
284
285static void r3k_flush_cache_sigtramp(unsigned long addr)
286{
287 unsigned long flags;
288
289#ifdef DEBUG_CACHE
290 printk("csigtramp[%08lx]", addr);
291#endif
292
293 flags = read_c0_status();
294
295 write_c0_status(flags&~ST0_IEC);
296
297 /* Fill the TLB to avoid an exception with caches isolated. */
298 asm ( "lw\t$0, 0x000(%0)\n\t"
299 "lw\t$0, 0x004(%0)\n\t"
300 : : "r" (addr) );
301
302 write_c0_status((ST0_ISC|ST0_SWC|flags)&~ST0_IEC);
303
304 asm ( "sb\t$0, 0x000(%0)\n\t"
305 "sb\t$0, 0x004(%0)\n\t"
306 : : "r" (addr) );
307
308 write_c0_status(flags);
309}
310
311static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size)
312{
313 /* Catch bad driver code */
314 BUG_ON(size == 0);
315
316 iob();
317 r3k_flush_dcache_range(start, start + size);
318}
319
320void __init ld_mmu_r23000(void)
321{
322 extern void build_clear_page(void);
323 extern void build_copy_page(void);
324
325 r3k_probe_cache();
326
327 flush_cache_all = r3k_flush_cache_all;
328 __flush_cache_all = r3k___flush_cache_all;
329 flush_cache_mm = r3k_flush_cache_mm;
330 flush_cache_range = r3k_flush_cache_range;
331 flush_cache_page = r3k_flush_cache_page;
332 flush_icache_page = r3k_flush_icache_page;
333 flush_icache_range = r3k_flush_icache_range;
334
335 flush_cache_sigtramp = r3k_flush_cache_sigtramp;
336 flush_data_cache_page = r3k_flush_data_cache_page;
337
338 _dma_cache_wback_inv = r3k_dma_cache_wback_inv;
339 _dma_cache_wback = r3k_dma_cache_wback_inv;
340 _dma_cache_inv = r3k_dma_cache_wback_inv;
341
342 printk("Primary instruction cache %ldkB, linesize %ld bytes.\n",
343 icache_size >> 10, icache_lsize);
344 printk("Primary data cache %ldkB, linesize %ld bytes.\n",
345 dcache_size >> 10, dcache_lsize);
346
347 build_clear_page();
348 build_copy_page();
349}
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
new file mode 100644
index 000000000000..a03ebb2cba67
--- /dev/null
+++ b/arch/mips/mm/c-r4k.c
@@ -0,0 +1,1260 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
9 */
10#include <linux/config.h>
11#include <linux/init.h>
12#include <linux/kernel.h>
13#include <linux/sched.h>
14#include <linux/mm.h>
15#include <linux/bitops.h>
16
17#include <asm/bcache.h>
18#include <asm/bootinfo.h>
19#include <asm/cacheops.h>
20#include <asm/cpu.h>
21#include <asm/cpu-features.h>
22#include <asm/io.h>
23#include <asm/page.h>
24#include <asm/pgtable.h>
25#include <asm/r4kcache.h>
26#include <asm/system.h>
27#include <asm/mmu_context.h>
28#include <asm/war.h>
29
30static unsigned long icache_size, dcache_size, scache_size;
31
32/*
33 * Dummy cache handling routines for machines without boardcaches
34 */
35static void no_sc_noop(void) {}
36
37static struct bcache_ops no_sc_ops = {
38 .bc_enable = (void *)no_sc_noop,
39 .bc_disable = (void *)no_sc_noop,
40 .bc_wback_inv = (void *)no_sc_noop,
41 .bc_inv = (void *)no_sc_noop
42};
43
44struct bcache_ops *bcops = &no_sc_ops;
45
46#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x2010)
47#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x2020)
48
49#define R4600_HIT_CACHEOP_WAR_IMPL \
50do { \
51 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \
52 *(volatile unsigned long *)CKSEG1; \
53 if (R4600_V1_HIT_CACHEOP_WAR) \
54 __asm__ __volatile__("nop;nop;nop;nop"); \
55} while (0)
56
57static void (*r4k_blast_dcache_page)(unsigned long addr);
58
59static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
60{
61 R4600_HIT_CACHEOP_WAR_IMPL;
62 blast_dcache32_page(addr);
63}
64
65static inline void r4k_blast_dcache_page_setup(void)
66{
67 unsigned long dc_lsize = cpu_dcache_line_size();
68
69 if (dc_lsize == 16)
70 r4k_blast_dcache_page = blast_dcache16_page;
71 else if (dc_lsize == 32)
72 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
73}
74
75static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
76
77static inline void r4k_blast_dcache_page_indexed_setup(void)
78{
79 unsigned long dc_lsize = cpu_dcache_line_size();
80
81 if (dc_lsize == 16)
82 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
83 else if (dc_lsize == 32)
84 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
85}
86
87static void (* r4k_blast_dcache)(void);
88
89static inline void r4k_blast_dcache_setup(void)
90{
91 unsigned long dc_lsize = cpu_dcache_line_size();
92
93 if (dc_lsize == 16)
94 r4k_blast_dcache = blast_dcache16;
95 else if (dc_lsize == 32)
96 r4k_blast_dcache = blast_dcache32;
97}
98
99/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
100#define JUMP_TO_ALIGN(order) \
101 __asm__ __volatile__( \
102 "b\t1f\n\t" \
103 ".align\t" #order "\n\t" \
104 "1:\n\t" \
105 )
106#define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
107#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
108
109static inline void blast_r4600_v1_icache32(void)
110{
111 unsigned long flags;
112
113 local_irq_save(flags);
114 blast_icache32();
115 local_irq_restore(flags);
116}
117
118static inline void tx49_blast_icache32(void)
119{
120 unsigned long start = INDEX_BASE;
121 unsigned long end = start + current_cpu_data.icache.waysize;
122 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
123 unsigned long ws_end = current_cpu_data.icache.ways <<
124 current_cpu_data.icache.waybit;
125 unsigned long ws, addr;
126
127 CACHE32_UNROLL32_ALIGN2;
128 /* I'm in even chunk. blast odd chunks */
129 for (ws = 0; ws < ws_end; ws += ws_inc)
130 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
131 cache32_unroll32(addr|ws,Index_Invalidate_I);
132 CACHE32_UNROLL32_ALIGN;
133 /* I'm in odd chunk. blast even chunks */
134 for (ws = 0; ws < ws_end; ws += ws_inc)
135 for (addr = start; addr < end; addr += 0x400 * 2)
136 cache32_unroll32(addr|ws,Index_Invalidate_I);
137}
138
139static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
140{
141 unsigned long flags;
142
143 local_irq_save(flags);
144 blast_icache32_page_indexed(page);
145 local_irq_restore(flags);
146}
147
148static inline void tx49_blast_icache32_page_indexed(unsigned long page)
149{
150 unsigned long start = page;
151 unsigned long end = start + PAGE_SIZE;
152 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
153 unsigned long ws_end = current_cpu_data.icache.ways <<
154 current_cpu_data.icache.waybit;
155 unsigned long ws, addr;
156
157 CACHE32_UNROLL32_ALIGN2;
158 /* I'm in even chunk. blast odd chunks */
159 for (ws = 0; ws < ws_end; ws += ws_inc)
160 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
161 cache32_unroll32(addr|ws,Index_Invalidate_I);
162 CACHE32_UNROLL32_ALIGN;
163 /* I'm in odd chunk. blast even chunks */
164 for (ws = 0; ws < ws_end; ws += ws_inc)
165 for (addr = start; addr < end; addr += 0x400 * 2)
166 cache32_unroll32(addr|ws,Index_Invalidate_I);
167}
168
169static void (* r4k_blast_icache_page)(unsigned long addr);
170
171static inline void r4k_blast_icache_page_setup(void)
172{
173 unsigned long ic_lsize = cpu_icache_line_size();
174
175 if (ic_lsize == 16)
176 r4k_blast_icache_page = blast_icache16_page;
177 else if (ic_lsize == 32)
178 r4k_blast_icache_page = blast_icache32_page;
179 else if (ic_lsize == 64)
180 r4k_blast_icache_page = blast_icache64_page;
181}
182
183
184static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
185
186static inline void r4k_blast_icache_page_indexed_setup(void)
187{
188 unsigned long ic_lsize = cpu_icache_line_size();
189
190 if (ic_lsize == 16)
191 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
192 else if (ic_lsize == 32) {
193 if (TX49XX_ICACHE_INDEX_INV_WAR)
194 r4k_blast_icache_page_indexed =
195 tx49_blast_icache32_page_indexed;
196 else if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
197 r4k_blast_icache_page_indexed =
198 blast_icache32_r4600_v1_page_indexed;
199 else
200 r4k_blast_icache_page_indexed =
201 blast_icache32_page_indexed;
202 } else if (ic_lsize == 64)
203 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
204}
205
206static void (* r4k_blast_icache)(void);
207
208static inline void r4k_blast_icache_setup(void)
209{
210 unsigned long ic_lsize = cpu_icache_line_size();
211
212 if (ic_lsize == 16)
213 r4k_blast_icache = blast_icache16;
214 else if (ic_lsize == 32) {
215 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
216 r4k_blast_icache = blast_r4600_v1_icache32;
217 else if (TX49XX_ICACHE_INDEX_INV_WAR)
218 r4k_blast_icache = tx49_blast_icache32;
219 else
220 r4k_blast_icache = blast_icache32;
221 } else if (ic_lsize == 64)
222 r4k_blast_icache = blast_icache64;
223}
224
225static void (* r4k_blast_scache_page)(unsigned long addr);
226
227static inline void r4k_blast_scache_page_setup(void)
228{
229 unsigned long sc_lsize = cpu_scache_line_size();
230
231 if (sc_lsize == 16)
232 r4k_blast_scache_page = blast_scache16_page;
233 else if (sc_lsize == 32)
234 r4k_blast_scache_page = blast_scache32_page;
235 else if (sc_lsize == 64)
236 r4k_blast_scache_page = blast_scache64_page;
237 else if (sc_lsize == 128)
238 r4k_blast_scache_page = blast_scache128_page;
239}
240
241static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
242
243static inline void r4k_blast_scache_page_indexed_setup(void)
244{
245 unsigned long sc_lsize = cpu_scache_line_size();
246
247 if (sc_lsize == 16)
248 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
249 else if (sc_lsize == 32)
250 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
251 else if (sc_lsize == 64)
252 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
253 else if (sc_lsize == 128)
254 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
255}
256
257static void (* r4k_blast_scache)(void);
258
259static inline void r4k_blast_scache_setup(void)
260{
261 unsigned long sc_lsize = cpu_scache_line_size();
262
263 if (sc_lsize == 16)
264 r4k_blast_scache = blast_scache16;
265 else if (sc_lsize == 32)
266 r4k_blast_scache = blast_scache32;
267 else if (sc_lsize == 64)
268 r4k_blast_scache = blast_scache64;
269 else if (sc_lsize == 128)
270 r4k_blast_scache = blast_scache128;
271}
272
273/*
274 * This is former mm's flush_cache_all() which really should be
275 * flush_cache_vunmap these days ...
276 */
277static inline void local_r4k_flush_cache_all(void * args)
278{
279 r4k_blast_dcache();
280 r4k_blast_icache();
281}
282
283static void r4k_flush_cache_all(void)
284{
285 if (!cpu_has_dc_aliases)
286 return;
287
288 on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1);
289}
290
291static inline void local_r4k___flush_cache_all(void * args)
292{
293 r4k_blast_dcache();
294 r4k_blast_icache();
295
296 switch (current_cpu_data.cputype) {
297 case CPU_R4000SC:
298 case CPU_R4000MC:
299 case CPU_R4400SC:
300 case CPU_R4400MC:
301 case CPU_R10000:
302 case CPU_R12000:
303 r4k_blast_scache();
304 }
305}
306
307static void r4k___flush_cache_all(void)
308{
309 on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1);
310}
311
312static inline void local_r4k_flush_cache_range(void * args)
313{
314 struct vm_area_struct *vma = args;
315 int exec;
316
317 if (!(cpu_context(smp_processor_id(), vma->vm_mm)))
318 return;
319
320 exec = vma->vm_flags & VM_EXEC;
321 if (cpu_has_dc_aliases || exec)
322 r4k_blast_dcache();
323 if (exec)
324 r4k_blast_icache();
325}
326
327static void r4k_flush_cache_range(struct vm_area_struct *vma,
328 unsigned long start, unsigned long end)
329{
330 on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1);
331}
332
333static inline void local_r4k_flush_cache_mm(void * args)
334{
335 struct mm_struct *mm = args;
336
337 if (!cpu_context(smp_processor_id(), mm))
338 return;
339
340 r4k_blast_dcache();
341 r4k_blast_icache();
342
343 /*
344 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we
345 * only flush the primary caches but R10000 and R12000 behave sane ...
346 */
347 if (current_cpu_data.cputype == CPU_R4000SC ||
348 current_cpu_data.cputype == CPU_R4000MC ||
349 current_cpu_data.cputype == CPU_R4400SC ||
350 current_cpu_data.cputype == CPU_R4400MC)
351 r4k_blast_scache();
352}
353
354static void r4k_flush_cache_mm(struct mm_struct *mm)
355{
356 if (!cpu_has_dc_aliases)
357 return;
358
359 on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1);
360}
361
362struct flush_cache_page_args {
363 struct vm_area_struct *vma;
364 unsigned long page;
365};
366
367static inline void local_r4k_flush_cache_page(void *args)
368{
369 struct flush_cache_page_args *fcp_args = args;
370 struct vm_area_struct *vma = fcp_args->vma;
371 unsigned long page = fcp_args->page;
372 int exec = vma->vm_flags & VM_EXEC;
373 struct mm_struct *mm = vma->vm_mm;
374 pgd_t *pgdp;
375 pmd_t *pmdp;
376 pte_t *ptep;
377
378 page &= PAGE_MASK;
379 pgdp = pgd_offset(mm, page);
380 pmdp = pmd_offset(pgdp, page);
381 ptep = pte_offset(pmdp, page);
382
383 /*
384 * If the page isn't marked valid, the page cannot possibly be
385 * in the cache.
386 */
387 if (!(pte_val(*ptep) & _PAGE_PRESENT))
388 return;
389
390 /*
391 * Doing flushes for another ASID than the current one is
392 * too difficult since stupid R4k caches do a TLB translation
393 * for every cache flush operation. So we do indexed flushes
394 * in that case, which doesn't overly flush the cache too much.
395 */
396 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
397 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
398 r4k_blast_dcache_page(page);
399 if (exec && !cpu_icache_snoops_remote_store)
400 r4k_blast_scache_page(page);
401 }
402 if (exec)
403 r4k_blast_icache_page(page);
404
405 return;
406 }
407
408 /*
409 * Do indexed flush, too much work to get the (possible) TLB refills
410 * to work correctly.
411 */
412 page = INDEX_BASE + (page & (dcache_size - 1));
413 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
414 r4k_blast_dcache_page_indexed(page);
415 if (exec && !cpu_icache_snoops_remote_store)
416 r4k_blast_scache_page_indexed(page);
417 }
418 if (exec) {
419 if (cpu_has_vtag_icache) {
420 int cpu = smp_processor_id();
421
422 if (cpu_context(cpu, vma->vm_mm) != 0)
423 drop_mmu_context(vma->vm_mm, cpu);
424 } else
425 r4k_blast_icache_page_indexed(page);
426 }
427}
428
429static void r4k_flush_cache_page(struct vm_area_struct *vma, unsigned long page, unsigned long pfn)
430{
431 struct flush_cache_page_args args;
432
433 /*
434 * If ownes no valid ASID yet, cannot possibly have gotten
435 * this page into the cache.
436 */
437 if (cpu_context(smp_processor_id(), vma->vm_mm) == 0)
438 return;
439
440 args.vma = vma;
441 args.page = page;
442
443 on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1);
444}
445
446static inline void local_r4k_flush_data_cache_page(void * addr)
447{
448 r4k_blast_dcache_page((unsigned long) addr);
449}
450
451static void r4k_flush_data_cache_page(unsigned long addr)
452{
453 on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1);
454}
455
456struct flush_icache_range_args {
457 unsigned long start;
458 unsigned long end;
459};
460
461static inline void local_r4k_flush_icache_range(void *args)
462{
463 struct flush_icache_range_args *fir_args = args;
464 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
465 unsigned long ic_lsize = current_cpu_data.icache.linesz;
466 unsigned long sc_lsize = current_cpu_data.scache.linesz;
467 unsigned long start = fir_args->start;
468 unsigned long end = fir_args->end;
469 unsigned long addr, aend;
470
471 if (!cpu_has_ic_fills_f_dc) {
472 if (end - start > dcache_size) {
473 r4k_blast_dcache();
474 } else {
475 addr = start & ~(dc_lsize - 1);
476 aend = (end - 1) & ~(dc_lsize - 1);
477
478 while (1) {
479 /* Hit_Writeback_Inv_D */
480 protected_writeback_dcache_line(addr);
481 if (addr == aend)
482 break;
483 addr += dc_lsize;
484 }
485 }
486
487 if (!cpu_icache_snoops_remote_store) {
488 if (end - start > scache_size) {
489 r4k_blast_scache();
490 } else {
491 addr = start & ~(sc_lsize - 1);
492 aend = (end - 1) & ~(sc_lsize - 1);
493
494 while (1) {
495 /* Hit_Writeback_Inv_D */
496 protected_writeback_scache_line(addr);
497 if (addr == aend)
498 break;
499 addr += sc_lsize;
500 }
501 }
502 }
503 }
504
505 if (end - start > icache_size)
506 r4k_blast_icache();
507 else {
508 addr = start & ~(ic_lsize - 1);
509 aend = (end - 1) & ~(ic_lsize - 1);
510 while (1) {
511 /* Hit_Invalidate_I */
512 protected_flush_icache_line(addr);
513 if (addr == aend)
514 break;
515 addr += ic_lsize;
516 }
517 }
518}
519
520static void r4k_flush_icache_range(unsigned long start, unsigned long end)
521{
522 struct flush_icache_range_args args;
523
524 args.start = start;
525 args.end = end;
526
527 on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1);
528}
529
530/*
531 * Ok, this seriously sucks. We use them to flush a user page but don't
532 * know the virtual address, so we have to blast away the whole icache
533 * which is significantly more expensive than the real thing. Otoh we at
534 * least know the kernel address of the page so we can flush it
535 * selectivly.
536 */
537
538struct flush_icache_page_args {
539 struct vm_area_struct *vma;
540 struct page *page;
541};
542
543static inline void local_r4k_flush_icache_page(void *args)
544{
545 struct flush_icache_page_args *fip_args = args;
546 struct vm_area_struct *vma = fip_args->vma;
547 struct page *page = fip_args->page;
548
549 /*
550 * Tricky ... Because we don't know the virtual address we've got the
551 * choice of either invalidating the entire primary and secondary
552 * caches or invalidating the secondary caches also. With the subset
553 * enforcment on R4000SC, R4400SC, R10000 and R12000 invalidating the
554 * secondary cache will result in any entries in the primary caches
555 * also getting invalidated which hopefully is a bit more economical.
556 */
557 if (cpu_has_subset_pcaches) {
558 unsigned long addr = (unsigned long) page_address(page);
559
560 r4k_blast_scache_page(addr);
561 ClearPageDcacheDirty(page);
562
563 return;
564 }
565
566 if (!cpu_has_ic_fills_f_dc) {
567 unsigned long addr = (unsigned long) page_address(page);
568 r4k_blast_dcache_page(addr);
569 if (!cpu_icache_snoops_remote_store)
570 r4k_blast_scache_page(addr);
571 ClearPageDcacheDirty(page);
572 }
573
574 /*
575 * We're not sure of the virtual address(es) involved here, so
576 * we have to flush the entire I-cache.
577 */
578 if (cpu_has_vtag_icache) {
579 int cpu = smp_processor_id();
580
581 if (cpu_context(cpu, vma->vm_mm) != 0)
582 drop_mmu_context(vma->vm_mm, cpu);
583 } else
584 r4k_blast_icache();
585}
586
587static void r4k_flush_icache_page(struct vm_area_struct *vma,
588 struct page *page)
589{
590 struct flush_icache_page_args args;
591
592 /*
593 * If there's no context yet, or the page isn't executable, no I-cache
594 * flush is needed.
595 */
596 if (!(vma->vm_flags & VM_EXEC))
597 return;
598
599 args.vma = vma;
600 args.page = page;
601
602 on_each_cpu(local_r4k_flush_icache_page, &args, 1, 1);
603}
604
605
606#ifdef CONFIG_DMA_NONCOHERENT
607
608static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
609{
610 unsigned long end, a;
611
612 /* Catch bad driver code */
613 BUG_ON(size == 0);
614
615 if (cpu_has_subset_pcaches) {
616 unsigned long sc_lsize = current_cpu_data.scache.linesz;
617
618 if (size >= scache_size) {
619 r4k_blast_scache();
620 return;
621 }
622
623 a = addr & ~(sc_lsize - 1);
624 end = (addr + size - 1) & ~(sc_lsize - 1);
625 while (1) {
626 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
627 if (a == end)
628 break;
629 a += sc_lsize;
630 }
631 return;
632 }
633
634 /*
635 * Either no secondary cache or the available caches don't have the
636 * subset property so we have to flush the primary caches
637 * explicitly
638 */
639 if (size >= dcache_size) {
640 r4k_blast_dcache();
641 } else {
642 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
643
644 R4600_HIT_CACHEOP_WAR_IMPL;
645 a = addr & ~(dc_lsize - 1);
646 end = (addr + size - 1) & ~(dc_lsize - 1);
647 while (1) {
648 flush_dcache_line(a); /* Hit_Writeback_Inv_D */
649 if (a == end)
650 break;
651 a += dc_lsize;
652 }
653 }
654
655 bc_wback_inv(addr, size);
656}
657
658static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
659{
660 unsigned long end, a;
661
662 /* Catch bad driver code */
663 BUG_ON(size == 0);
664
665 if (cpu_has_subset_pcaches) {
666 unsigned long sc_lsize = current_cpu_data.scache.linesz;
667
668 if (size >= scache_size) {
669 r4k_blast_scache();
670 return;
671 }
672
673 a = addr & ~(sc_lsize - 1);
674 end = (addr + size - 1) & ~(sc_lsize - 1);
675 while (1) {
676 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
677 if (a == end)
678 break;
679 a += sc_lsize;
680 }
681 return;
682 }
683
684 if (size >= dcache_size) {
685 r4k_blast_dcache();
686 } else {
687 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
688
689 R4600_HIT_CACHEOP_WAR_IMPL;
690 a = addr & ~(dc_lsize - 1);
691 end = (addr + size - 1) & ~(dc_lsize - 1);
692 while (1) {
693 flush_dcache_line(a); /* Hit_Writeback_Inv_D */
694 if (a == end)
695 break;
696 a += dc_lsize;
697 }
698 }
699
700 bc_inv(addr, size);
701}
702#endif /* CONFIG_DMA_NONCOHERENT */
703
704/*
705 * While we're protected against bad userland addresses we don't care
706 * very much about what happens in that case. Usually a segmentation
707 * fault will dump the process later on anyway ...
708 */
709static void local_r4k_flush_cache_sigtramp(void * arg)
710{
711 unsigned long ic_lsize = current_cpu_data.icache.linesz;
712 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
713 unsigned long sc_lsize = current_cpu_data.scache.linesz;
714 unsigned long addr = (unsigned long) arg;
715
716 R4600_HIT_CACHEOP_WAR_IMPL;
717 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
718 if (!cpu_icache_snoops_remote_store)
719 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
720 protected_flush_icache_line(addr & ~(ic_lsize - 1));
721 if (MIPS4K_ICACHE_REFILL_WAR) {
722 __asm__ __volatile__ (
723 ".set push\n\t"
724 ".set noat\n\t"
725 ".set mips3\n\t"
726#ifdef CONFIG_MIPS32
727 "la $at,1f\n\t"
728#endif
729#ifdef CONFIG_MIPS64
730 "dla $at,1f\n\t"
731#endif
732 "cache %0,($at)\n\t"
733 "nop; nop; nop\n"
734 "1:\n\t"
735 ".set pop"
736 :
737 : "i" (Hit_Invalidate_I));
738 }
739 if (MIPS_CACHE_SYNC_WAR)
740 __asm__ __volatile__ ("sync");
741}
742
743static void r4k_flush_cache_sigtramp(unsigned long addr)
744{
745 on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1);
746}
747
748static void r4k_flush_icache_all(void)
749{
750 if (cpu_has_vtag_icache)
751 r4k_blast_icache();
752}
753
754static inline void rm7k_erratum31(void)
755{
756 const unsigned long ic_lsize = 32;
757 unsigned long addr;
758
759 /* RM7000 erratum #31. The icache is screwed at startup. */
760 write_c0_taglo(0);
761 write_c0_taghi(0);
762
763 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
764 __asm__ __volatile__ (
765 ".set noreorder\n\t"
766 ".set mips3\n\t"
767 "cache\t%1, 0(%0)\n\t"
768 "cache\t%1, 0x1000(%0)\n\t"
769 "cache\t%1, 0x2000(%0)\n\t"
770 "cache\t%1, 0x3000(%0)\n\t"
771 "cache\t%2, 0(%0)\n\t"
772 "cache\t%2, 0x1000(%0)\n\t"
773 "cache\t%2, 0x2000(%0)\n\t"
774 "cache\t%2, 0x3000(%0)\n\t"
775 "cache\t%1, 0(%0)\n\t"
776 "cache\t%1, 0x1000(%0)\n\t"
777 "cache\t%1, 0x2000(%0)\n\t"
778 "cache\t%1, 0x3000(%0)\n\t"
779 ".set\tmips0\n\t"
780 ".set\treorder\n\t"
781 :
782 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
783 }
784}
785
786static char *way_string[] __initdata = { NULL, "direct mapped", "2-way",
787 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
788};
789
790static void __init probe_pcache(void)
791{
792 struct cpuinfo_mips *c = &current_cpu_data;
793 unsigned int config = read_c0_config();
794 unsigned int prid = read_c0_prid();
795 unsigned long config1;
796 unsigned int lsize;
797
798 switch (c->cputype) {
799 case CPU_R4600: /* QED style two way caches? */
800 case CPU_R4700:
801 case CPU_R5000:
802 case CPU_NEVADA:
803 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
804 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
805 c->icache.ways = 2;
806 c->icache.waybit = ffs(icache_size/2) - 1;
807
808 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
809 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
810 c->dcache.ways = 2;
811 c->dcache.waybit= ffs(dcache_size/2) - 1;
812
813 c->options |= MIPS_CPU_CACHE_CDEX_P;
814 break;
815
816 case CPU_R5432:
817 case CPU_R5500:
818 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
819 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
820 c->icache.ways = 2;
821 c->icache.waybit= 0;
822
823 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
824 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
825 c->dcache.ways = 2;
826 c->dcache.waybit = 0;
827
828 c->options |= MIPS_CPU_CACHE_CDEX_P;
829 break;
830
831 case CPU_TX49XX:
832 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
833 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
834 c->icache.ways = 4;
835 c->icache.waybit= 0;
836
837 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
838 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
839 c->dcache.ways = 4;
840 c->dcache.waybit = 0;
841
842 c->options |= MIPS_CPU_CACHE_CDEX_P;
843 break;
844
845 case CPU_R4000PC:
846 case CPU_R4000SC:
847 case CPU_R4000MC:
848 case CPU_R4400PC:
849 case CPU_R4400SC:
850 case CPU_R4400MC:
851 case CPU_R4300:
852 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
853 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
854 c->icache.ways = 1;
855 c->icache.waybit = 0; /* doesn't matter */
856
857 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
858 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
859 c->dcache.ways = 1;
860 c->dcache.waybit = 0; /* does not matter */
861
862 c->options |= MIPS_CPU_CACHE_CDEX_P;
863 break;
864
865 case CPU_R10000:
866 case CPU_R12000:
867 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
868 c->icache.linesz = 64;
869 c->icache.ways = 2;
870 c->icache.waybit = 0;
871
872 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
873 c->dcache.linesz = 32;
874 c->dcache.ways = 2;
875 c->dcache.waybit = 0;
876
877 c->options |= MIPS_CPU_PREFETCH;
878 break;
879
880 case CPU_VR4133:
881 write_c0_config(config & ~CONF_EB);
882 case CPU_VR4131:
883 /* Workaround for cache instruction bug of VR4131 */
884 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
885 c->processor_id == 0x0c82U) {
886 config &= ~0x00000030U;
887 config |= 0x00410000U;
888 write_c0_config(config);
889 }
890 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
891 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
892 c->icache.ways = 2;
893 c->icache.waybit = ffs(icache_size/2) - 1;
894
895 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
896 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
897 c->dcache.ways = 2;
898 c->dcache.waybit = ffs(dcache_size/2) - 1;
899
900 c->options |= MIPS_CPU_CACHE_CDEX_P;
901 break;
902
903 case CPU_VR41XX:
904 case CPU_VR4111:
905 case CPU_VR4121:
906 case CPU_VR4122:
907 case CPU_VR4181:
908 case CPU_VR4181A:
909 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
910 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
911 c->icache.ways = 1;
912 c->icache.waybit = 0; /* doesn't matter */
913
914 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
915 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
916 c->dcache.ways = 1;
917 c->dcache.waybit = 0; /* does not matter */
918
919 c->options |= MIPS_CPU_CACHE_CDEX_P;
920 break;
921
922 case CPU_RM7000:
923 rm7k_erratum31();
924
925 case CPU_RM9000:
926 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
927 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
928 c->icache.ways = 4;
929 c->icache.waybit = ffs(icache_size / c->icache.ways) - 1;
930
931 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
932 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
933 c->dcache.ways = 4;
934 c->dcache.waybit = ffs(dcache_size / c->dcache.ways) - 1;
935
936#if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR)
937 c->options |= MIPS_CPU_CACHE_CDEX_P;
938#endif
939 c->options |= MIPS_CPU_PREFETCH;
940 break;
941
942 default:
943 if (!(config & MIPS_CONF_M))
944 panic("Don't know how to probe P-caches on this cpu.");
945
946 /*
947 * So we seem to be a MIPS32 or MIPS64 CPU
948 * So let's probe the I-cache ...
949 */
950 config1 = read_c0_config1();
951
952 if ((lsize = ((config1 >> 19) & 7)))
953 c->icache.linesz = 2 << lsize;
954 else
955 c->icache.linesz = lsize;
956 c->icache.sets = 64 << ((config1 >> 22) & 7);
957 c->icache.ways = 1 + ((config1 >> 16) & 7);
958
959 icache_size = c->icache.sets *
960 c->icache.ways *
961 c->icache.linesz;
962 c->icache.waybit = ffs(icache_size/c->icache.ways) - 1;
963
964 if (config & 0x8) /* VI bit */
965 c->icache.flags |= MIPS_CACHE_VTAG;
966
967 /*
968 * Now probe the MIPS32 / MIPS64 data cache.
969 */
970 c->dcache.flags = 0;
971
972 if ((lsize = ((config1 >> 10) & 7)))
973 c->dcache.linesz = 2 << lsize;
974 else
975 c->dcache.linesz= lsize;
976 c->dcache.sets = 64 << ((config1 >> 13) & 7);
977 c->dcache.ways = 1 + ((config1 >> 7) & 7);
978
979 dcache_size = c->dcache.sets *
980 c->dcache.ways *
981 c->dcache.linesz;
982 c->dcache.waybit = ffs(dcache_size/c->dcache.ways) - 1;
983
984 c->options |= MIPS_CPU_PREFETCH;
985 break;
986 }
987
988 /*
989 * Processor configuration sanity check for the R4000SC erratum
990 * #5. With page sizes larger than 32kB there is no possibility
991 * to get a VCE exception anymore so we don't care about this
992 * misconfiguration. The case is rather theoretical anyway;
993 * presumably no vendor is shipping his hardware in the "bad"
994 * configuration.
995 */
996 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
997 !(config & CONF_SC) && c->icache.linesz != 16 &&
998 PAGE_SIZE <= 0x8000)
999 panic("Improper R4000SC processor configuration detected");
1000
1001 /* compute a couple of other cache variables */
1002 c->icache.waysize = icache_size / c->icache.ways;
1003 c->dcache.waysize = dcache_size / c->dcache.ways;
1004
1005 c->icache.sets = icache_size / (c->icache.linesz * c->icache.ways);
1006 c->dcache.sets = dcache_size / (c->dcache.linesz * c->dcache.ways);
1007
1008 /*
1009 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB
1010 * 2-way virtually indexed so normally would suffer from aliases. So
1011 * normally they'd suffer from aliases but magic in the hardware deals
1012 * with that for us so we don't need to take care ourselves.
1013 */
1014 if (c->cputype != CPU_R10000 && c->cputype != CPU_R12000)
1015 if (c->dcache.waysize > PAGE_SIZE)
1016 c->dcache.flags |= MIPS_CACHE_ALIASES;
1017
1018 switch (c->cputype) {
1019 case CPU_20KC:
1020 /*
1021 * Some older 20Kc chips doesn't have the 'VI' bit in
1022 * the config register.
1023 */
1024 c->icache.flags |= MIPS_CACHE_VTAG;
1025 break;
1026
1027 case CPU_AU1500:
1028 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1029 break;
1030 }
1031
1032 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1033 icache_size >> 10,
1034 cpu_has_vtag_icache ? "virtually tagged" : "physically tagged",
1035 way_string[c->icache.ways], c->icache.linesz);
1036
1037 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n",
1038 dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz);
1039}
1040
1041/*
1042 * If you even _breathe_ on this function, look at the gcc output and make sure
1043 * it does not pop things on and off the stack for the cache sizing loop that
1044 * executes in KSEG1 space or else you will crash and burn badly. You have
1045 * been warned.
1046 */
1047static int __init probe_scache(void)
1048{
1049 extern unsigned long stext;
1050 unsigned long flags, addr, begin, end, pow2;
1051 unsigned int config = read_c0_config();
1052 struct cpuinfo_mips *c = &current_cpu_data;
1053 int tmp;
1054
1055 if (config & CONF_SC)
1056 return 0;
1057
1058 begin = (unsigned long) &stext;
1059 begin &= ~((4 * 1024 * 1024) - 1);
1060 end = begin + (4 * 1024 * 1024);
1061
1062 /*
1063 * This is such a bitch, you'd think they would make it easy to do
1064 * this. Away you daemons of stupidity!
1065 */
1066 local_irq_save(flags);
1067
1068 /* Fill each size-multiple cache line with a valid tag. */
1069 pow2 = (64 * 1024);
1070 for (addr = begin; addr < end; addr = (begin + pow2)) {
1071 unsigned long *p = (unsigned long *) addr;
1072 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1073 pow2 <<= 1;
1074 }
1075
1076 /* Load first line with zero (therefore invalid) tag. */
1077 write_c0_taglo(0);
1078 write_c0_taghi(0);
1079 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1080 cache_op(Index_Store_Tag_I, begin);
1081 cache_op(Index_Store_Tag_D, begin);
1082 cache_op(Index_Store_Tag_SD, begin);
1083
1084 /* Now search for the wrap around point. */
1085 pow2 = (128 * 1024);
1086 tmp = 0;
1087 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1088 cache_op(Index_Load_Tag_SD, addr);
1089 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1090 if (!read_c0_taglo())
1091 break;
1092 pow2 <<= 1;
1093 }
1094 local_irq_restore(flags);
1095 addr -= begin;
1096
1097 scache_size = addr;
1098 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1099 c->scache.ways = 1;
1100 c->dcache.waybit = 0; /* does not matter */
1101
1102 return 1;
1103}
1104
1105typedef int (*probe_func_t)(unsigned long);
1106extern int r5k_sc_init(void);
1107extern int rm7k_sc_init(void);
1108
1109static void __init setup_scache(void)
1110{
1111 struct cpuinfo_mips *c = &current_cpu_data;
1112 unsigned int config = read_c0_config();
1113 probe_func_t probe_scache_kseg1;
1114 int sc_present = 0;
1115
1116 /*
1117 * Do the probing thing on R4000SC and R4400SC processors. Other
1118 * processors don't have a S-cache that would be relevant to the
1119 * Linux memory managment.
1120 */
1121 switch (c->cputype) {
1122 case CPU_R4000SC:
1123 case CPU_R4000MC:
1124 case CPU_R4400SC:
1125 case CPU_R4400MC:
1126 probe_scache_kseg1 = (probe_func_t) (CKSEG1ADDR(&probe_scache));
1127 sc_present = probe_scache_kseg1(config);
1128 if (sc_present)
1129 c->options |= MIPS_CPU_CACHE_CDEX_S;
1130 break;
1131
1132 case CPU_R10000:
1133 case CPU_R12000:
1134 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1135 c->scache.linesz = 64 << ((config >> 13) & 1);
1136 c->scache.ways = 2;
1137 c->scache.waybit= 0;
1138 sc_present = 1;
1139 break;
1140
1141 case CPU_R5000:
1142 case CPU_NEVADA:
1143#ifdef CONFIG_R5000_CPU_SCACHE
1144 r5k_sc_init();
1145#endif
1146 return;
1147
1148 case CPU_RM7000:
1149 case CPU_RM9000:
1150#ifdef CONFIG_RM7000_CPU_SCACHE
1151 rm7k_sc_init();
1152#endif
1153 return;
1154
1155 default:
1156 sc_present = 0;
1157 }
1158
1159 if (!sc_present)
1160 return;
1161
1162 if ((c->isa_level == MIPS_CPU_ISA_M32 ||
1163 c->isa_level == MIPS_CPU_ISA_M64) &&
1164 !(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1165 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1166
1167 /* compute a couple of other cache variables */
1168 c->scache.waysize = scache_size / c->scache.ways;
1169
1170 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1171
1172 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1173 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1174
1175 c->options |= MIPS_CPU_SUBSET_CACHES;
1176}
1177
1178static inline void coherency_setup(void)
1179{
1180 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT);
1181
1182 /*
1183 * c0_status.cu=0 specifies that updates by the sc instruction use
1184 * the coherency mode specified by the TLB; 1 means cachable
1185 * coherent update on write will be used. Not all processors have
1186 * this bit and; some wire it to zero, others like Toshiba had the
1187 * silly idea of putting something else there ...
1188 */
1189 switch (current_cpu_data.cputype) {
1190 case CPU_R4000PC:
1191 case CPU_R4000SC:
1192 case CPU_R4000MC:
1193 case CPU_R4400PC:
1194 case CPU_R4400SC:
1195 case CPU_R4400MC:
1196 clear_c0_config(CONF_CU);
1197 break;
1198 }
1199}
1200
1201void __init ld_mmu_r4xx0(void)
1202{
1203 extern void build_clear_page(void);
1204 extern void build_copy_page(void);
1205 extern char except_vec2_generic;
1206 struct cpuinfo_mips *c = &current_cpu_data;
1207
1208 /* Default cache error handler for R4000 and R5000 family */
1209 memcpy((void *)(CAC_BASE + 0x100), &except_vec2_generic, 0x80);
1210 memcpy((void *)(UNCAC_BASE + 0x100), &except_vec2_generic, 0x80);
1211
1212 probe_pcache();
1213 setup_scache();
1214
1215 if (c->dcache.sets * c->dcache.ways > PAGE_SIZE)
1216 c->dcache.flags |= MIPS_CACHE_ALIASES;
1217
1218 r4k_blast_dcache_page_setup();
1219 r4k_blast_dcache_page_indexed_setup();
1220 r4k_blast_dcache_setup();
1221 r4k_blast_icache_page_setup();
1222 r4k_blast_icache_page_indexed_setup();
1223 r4k_blast_icache_setup();
1224 r4k_blast_scache_page_setup();
1225 r4k_blast_scache_page_indexed_setup();
1226 r4k_blast_scache_setup();
1227
1228 /*
1229 * Some MIPS32 and MIPS64 processors have physically indexed caches.
1230 * This code supports virtually indexed processors and will be
1231 * unnecessarily inefficient on physically indexed processors.
1232 */
1233 shm_align_mask = max_t( unsigned long,
1234 c->dcache.sets * c->dcache.linesz - 1,
1235 PAGE_SIZE - 1);
1236
1237 flush_cache_all = r4k_flush_cache_all;
1238 __flush_cache_all = r4k___flush_cache_all;
1239 flush_cache_mm = r4k_flush_cache_mm;
1240 flush_cache_page = r4k_flush_cache_page;
1241 flush_icache_page = r4k_flush_icache_page;
1242 flush_cache_range = r4k_flush_cache_range;
1243
1244 flush_cache_sigtramp = r4k_flush_cache_sigtramp;
1245 flush_icache_all = r4k_flush_icache_all;
1246 flush_data_cache_page = r4k_flush_data_cache_page;
1247 flush_icache_range = r4k_flush_icache_range;
1248
1249#ifdef CONFIG_DMA_NONCOHERENT
1250 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1251 _dma_cache_wback = r4k_dma_cache_wback_inv;
1252 _dma_cache_inv = r4k_dma_cache_inv;
1253#endif
1254
1255 __flush_cache_all();
1256 coherency_setup();
1257
1258 build_clear_page();
1259 build_copy_page();
1260}
diff --git a/arch/mips/mm/c-sb1.c b/arch/mips/mm/c-sb1.c
new file mode 100644
index 000000000000..ab30afd63b32
--- /dev/null
+++ b/arch/mips/mm/c-sb1.c
@@ -0,0 +1,558 @@
1/*
2 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
3 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org)
4 * Copyright (C) 2000, 2001, 2002, 2003 Broadcom Corporation
5 * Copyright (C) 2004 Maciej W. Rozycki
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20 */
21#include <linux/config.h>
22#include <linux/init.h>
23
24#include <asm/asm.h>
25#include <asm/bootinfo.h>
26#include <asm/cacheops.h>
27#include <asm/cpu.h>
28#include <asm/mipsregs.h>
29#include <asm/mmu_context.h>
30#include <asm/uaccess.h>
31
32extern void sb1_dma_init(void);
33
34/* These are probed at ld_mmu time */
35static unsigned long icache_size;
36static unsigned long dcache_size;
37
38static unsigned short icache_line_size;
39static unsigned short dcache_line_size;
40
41static unsigned int icache_index_mask;
42static unsigned int dcache_index_mask;
43
44static unsigned short icache_assoc;
45static unsigned short dcache_assoc;
46
47static unsigned short icache_sets;
48static unsigned short dcache_sets;
49
50static unsigned int icache_range_cutoff;
51static unsigned int dcache_range_cutoff;
52
53/*
54 * The dcache is fully coherent to the system, with one
55 * big caveat: the instruction stream. In other words,
56 * if we miss in the icache, and have dirty data in the
57 * L1 dcache, then we'll go out to memory (or the L2) and
58 * get the not-as-recent data.
59 *
60 * So the only time we have to flush the dcache is when
61 * we're flushing the icache. Since the L2 is fully
62 * coherent to everything, including I/O, we never have
63 * to flush it
64 */
65
66#define cache_set_op(op, addr) \
67 __asm__ __volatile__( \
68 " .set noreorder \n" \
69 " .set mips64\n\t \n" \
70 " cache %0, (0<<13)(%1) \n" \
71 " cache %0, (1<<13)(%1) \n" \
72 " cache %0, (2<<13)(%1) \n" \
73 " cache %0, (3<<13)(%1) \n" \
74 " .set mips0 \n" \
75 " .set reorder" \
76 : \
77 : "i" (op), "r" (addr))
78
79#define sync() \
80 __asm__ __volatile( \
81 " .set mips64\n\t \n" \
82 " sync \n" \
83 " .set mips0")
84
85#define mispredict() \
86 __asm__ __volatile__( \
87 " bnezl $0, 1f \n" /* Force mispredict */ \
88 "1: \n");
89
90/*
91 * Writeback and invalidate the entire dcache
92 */
93static inline void __sb1_writeback_inv_dcache_all(void)
94{
95 unsigned long addr = 0;
96
97 while (addr < dcache_line_size * dcache_sets) {
98 cache_set_op(Index_Writeback_Inv_D, addr);
99 addr += dcache_line_size;
100 }
101}
102
103/*
104 * Writeback and invalidate a range of the dcache. The addresses are
105 * virtual, and since we're using index ops and bit 12 is part of both
106 * the virtual frame and physical index, we have to clear both sets
107 * (bit 12 set and cleared).
108 */
109static inline void __sb1_writeback_inv_dcache_range(unsigned long start,
110 unsigned long end)
111{
112 unsigned long index;
113
114 start &= ~(dcache_line_size - 1);
115 end = (end + dcache_line_size - 1) & ~(dcache_line_size - 1);
116
117 while (start != end) {
118 index = start & dcache_index_mask;
119 cache_set_op(Index_Writeback_Inv_D, index);
120 cache_set_op(Index_Writeback_Inv_D, index ^ (1<<12));
121 start += dcache_line_size;
122 }
123 sync();
124}
125
126/*
127 * Writeback and invalidate a range of the dcache. With physical
128 * addresseses, we don't have to worry about possible bit 12 aliasing.
129 * XXXKW is it worth turning on KX and using hit ops with xkphys?
130 */
131static inline void __sb1_writeback_inv_dcache_phys_range(unsigned long start,
132 unsigned long end)
133{
134 start &= ~(dcache_line_size - 1);
135 end = (end + dcache_line_size - 1) & ~(dcache_line_size - 1);
136
137 while (start != end) {
138 cache_set_op(Index_Writeback_Inv_D, start & dcache_index_mask);
139 start += dcache_line_size;
140 }
141 sync();
142}
143
144
145/*
146 * Invalidate the entire icache
147 */
148static inline void __sb1_flush_icache_all(void)
149{
150 unsigned long addr = 0;
151
152 while (addr < icache_line_size * icache_sets) {
153 cache_set_op(Index_Invalidate_I, addr);
154 addr += icache_line_size;
155 }
156}
157
158/*
159 * Flush the icache for a given physical page. Need to writeback the
160 * dcache first, then invalidate the icache. If the page isn't
161 * executable, nothing is required.
162 */
163static void local_sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)
164{
165 int cpu = smp_processor_id();
166
167#ifndef CONFIG_SMP
168 if (!(vma->vm_flags & VM_EXEC))
169 return;
170#endif
171
172 __sb1_writeback_inv_dcache_range(addr, addr + PAGE_SIZE);
173
174 /*
175 * Bumping the ASID is probably cheaper than the flush ...
176 */
177 if (cpu_context(cpu, vma->vm_mm) != 0)
178 drop_mmu_context(vma->vm_mm, cpu);
179}
180
181#ifdef CONFIG_SMP
182struct flush_cache_page_args {
183 struct vm_area_struct *vma;
184 unsigned long addr;
185 unsigned long pfn;
186};
187
188static void sb1_flush_cache_page_ipi(void *info)
189{
190 struct flush_cache_page_args *args = info;
191
192 local_sb1_flush_cache_page(args->vma, args->addr, args->pfn);
193}
194
195/* Dirty dcache could be on another CPU, so do the IPIs */
196static void sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)
197{
198 struct flush_cache_page_args args;
199
200 if (!(vma->vm_flags & VM_EXEC))
201 return;
202
203 addr &= PAGE_MASK;
204 args.vma = vma;
205 args.addr = addr;
206 args.pfn = pfn;
207 on_each_cpu(sb1_flush_cache_page_ipi, (void *) &args, 1, 1);
208}
209#else
210void sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)
211 __attribute__((alias("local_sb1_flush_cache_page")));
212#endif
213
214/*
215 * Invalidate a range of the icache. The addresses are virtual, and
216 * the cache is virtually indexed and tagged. However, we don't
217 * necessarily have the right ASID context, so use index ops instead
218 * of hit ops.
219 */
220static inline void __sb1_flush_icache_range(unsigned long start,
221 unsigned long end)
222{
223 start &= ~(icache_line_size - 1);
224 end = (end + icache_line_size - 1) & ~(icache_line_size - 1);
225
226 while (start != end) {
227 cache_set_op(Index_Invalidate_I, start & icache_index_mask);
228 start += icache_line_size;
229 }
230 mispredict();
231 sync();
232}
233
234
235/*
236 * Invalidate all caches on this CPU
237 */
238static void local_sb1___flush_cache_all(void)
239{
240 __sb1_writeback_inv_dcache_all();
241 __sb1_flush_icache_all();
242}
243
244#ifdef CONFIG_SMP
245void sb1___flush_cache_all_ipi(void *ignored)
246 __attribute__((alias("local_sb1___flush_cache_all")));
247
248static void sb1___flush_cache_all(void)
249{
250 on_each_cpu(sb1___flush_cache_all_ipi, 0, 1, 1);
251}
252#else
253void sb1___flush_cache_all(void)
254 __attribute__((alias("local_sb1___flush_cache_all")));
255#endif
256
257/*
258 * When flushing a range in the icache, we have to first writeback
259 * the dcache for the same range, so new ifetches will see any
260 * data that was dirty in the dcache.
261 *
262 * The start/end arguments are Kseg addresses (possibly mapped Kseg).
263 */
264
265static void local_sb1_flush_icache_range(unsigned long start,
266 unsigned long end)
267{
268 /* Just wb-inv the whole dcache if the range is big enough */
269 if ((end - start) > dcache_range_cutoff)
270 __sb1_writeback_inv_dcache_all();
271 else
272 __sb1_writeback_inv_dcache_range(start, end);
273
274 /* Just flush the whole icache if the range is big enough */
275 if ((end - start) > icache_range_cutoff)
276 __sb1_flush_icache_all();
277 else
278 __sb1_flush_icache_range(start, end);
279}
280
281#ifdef CONFIG_SMP
282struct flush_icache_range_args {
283 unsigned long start;
284 unsigned long end;
285};
286
287static void sb1_flush_icache_range_ipi(void *info)
288{
289 struct flush_icache_range_args *args = info;
290
291 local_sb1_flush_icache_range(args->start, args->end);
292}
293
294void sb1_flush_icache_range(unsigned long start, unsigned long end)
295{
296 struct flush_icache_range_args args;
297
298 args.start = start;
299 args.end = end;
300 on_each_cpu(sb1_flush_icache_range_ipi, &args, 1, 1);
301}
302#else
303void sb1_flush_icache_range(unsigned long start, unsigned long end)
304 __attribute__((alias("local_sb1_flush_icache_range")));
305#endif
306
307/*
308 * Flush the icache for a given physical page. Need to writeback the
309 * dcache first, then invalidate the icache. If the page isn't
310 * executable, nothing is required.
311 */
312static void local_sb1_flush_icache_page(struct vm_area_struct *vma,
313 struct page *page)
314{
315 unsigned long start;
316 int cpu = smp_processor_id();
317
318#ifndef CONFIG_SMP
319 if (!(vma->vm_flags & VM_EXEC))
320 return;
321#endif
322
323 /* Need to writeback any dirty data for that page, we have the PA */
324 start = (unsigned long)(page-mem_map) << PAGE_SHIFT;
325 __sb1_writeback_inv_dcache_phys_range(start, start + PAGE_SIZE);
326 /*
327 * If there's a context, bump the ASID (cheaper than a flush,
328 * since we don't know VAs!)
329 */
330 if (cpu_context(cpu, vma->vm_mm) != 0) {
331 drop_mmu_context(vma->vm_mm, cpu);
332 }
333}
334
335#ifdef CONFIG_SMP
336struct flush_icache_page_args {
337 struct vm_area_struct *vma;
338 struct page *page;
339};
340
341static void sb1_flush_icache_page_ipi(void *info)
342{
343 struct flush_icache_page_args *args = info;
344 local_sb1_flush_icache_page(args->vma, args->page);
345}
346
347/* Dirty dcache could be on another CPU, so do the IPIs */
348static void sb1_flush_icache_page(struct vm_area_struct *vma,
349 struct page *page)
350{
351 struct flush_icache_page_args args;
352
353 if (!(vma->vm_flags & VM_EXEC))
354 return;
355 args.vma = vma;
356 args.page = page;
357 on_each_cpu(sb1_flush_icache_page_ipi, (void *) &args, 1, 1);
358}
359#else
360void sb1_flush_icache_page(struct vm_area_struct *vma, struct page *page)
361 __attribute__((alias("local_sb1_flush_icache_page")));
362#endif
363
364/*
365 * A signal trampoline must fit into a single cacheline.
366 */
367static void local_sb1_flush_cache_sigtramp(unsigned long addr)
368{
369 cache_set_op(Index_Writeback_Inv_D, addr & dcache_index_mask);
370 cache_set_op(Index_Writeback_Inv_D, (addr ^ (1<<12)) & dcache_index_mask);
371 cache_set_op(Index_Invalidate_I, addr & icache_index_mask);
372 mispredict();
373}
374
375#ifdef CONFIG_SMP
376static void sb1_flush_cache_sigtramp_ipi(void *info)
377{
378 unsigned long iaddr = (unsigned long) info;
379 local_sb1_flush_cache_sigtramp(iaddr);
380}
381
382static void sb1_flush_cache_sigtramp(unsigned long addr)
383{
384 on_each_cpu(sb1_flush_cache_sigtramp_ipi, (void *) addr, 1, 1);
385}
386#else
387void sb1_flush_cache_sigtramp(unsigned long addr)
388 __attribute__((alias("local_sb1_flush_cache_sigtramp")));
389#endif
390
391
392/*
393 * Anything that just flushes dcache state can be ignored, as we're always
394 * coherent in dcache space. This is just a dummy function that all the
395 * nop'ed routines point to
396 */
397static void sb1_nop(void)
398{
399}
400
401/*
402 * Cache set values (from the mips64 spec)
403 * 0 - 64
404 * 1 - 128
405 * 2 - 256
406 * 3 - 512
407 * 4 - 1024
408 * 5 - 2048
409 * 6 - 4096
410 * 7 - Reserved
411 */
412
413static unsigned int decode_cache_sets(unsigned int config_field)
414{
415 if (config_field == 7) {
416 /* JDCXXX - Find a graceful way to abort. */
417 return 0;
418 }
419 return (1<<(config_field + 6));
420}
421
422/*
423 * Cache line size values (from the mips64 spec)
424 * 0 - No cache present.
425 * 1 - 4 bytes
426 * 2 - 8 bytes
427 * 3 - 16 bytes
428 * 4 - 32 bytes
429 * 5 - 64 bytes
430 * 6 - 128 bytes
431 * 7 - Reserved
432 */
433
434static unsigned int decode_cache_line_size(unsigned int config_field)
435{
436 if (config_field == 0) {
437 return 0;
438 } else if (config_field == 7) {
439 /* JDCXXX - Find a graceful way to abort. */
440 return 0;
441 }
442 return (1<<(config_field + 1));
443}
444
445/*
446 * Relevant bits of the config1 register format (from the MIPS32/MIPS64 specs)
447 *
448 * 24:22 Icache sets per way
449 * 21:19 Icache line size
450 * 18:16 Icache Associativity
451 * 15:13 Dcache sets per way
452 * 12:10 Dcache line size
453 * 9:7 Dcache Associativity
454 */
455
456static char *way_string[] = {
457 "direct mapped", "2-way", "3-way", "4-way",
458 "5-way", "6-way", "7-way", "8-way",
459};
460
461static __init void probe_cache_sizes(void)
462{
463 u32 config1;
464
465 config1 = read_c0_config1();
466 icache_line_size = decode_cache_line_size((config1 >> 19) & 0x7);
467 dcache_line_size = decode_cache_line_size((config1 >> 10) & 0x7);
468 icache_sets = decode_cache_sets((config1 >> 22) & 0x7);
469 dcache_sets = decode_cache_sets((config1 >> 13) & 0x7);
470 icache_assoc = ((config1 >> 16) & 0x7) + 1;
471 dcache_assoc = ((config1 >> 7) & 0x7) + 1;
472 icache_size = icache_line_size * icache_sets * icache_assoc;
473 dcache_size = dcache_line_size * dcache_sets * dcache_assoc;
474 /* Need to remove non-index bits for index ops */
475 icache_index_mask = (icache_sets - 1) * icache_line_size;
476 dcache_index_mask = (dcache_sets - 1) * dcache_line_size;
477 /*
478 * These are for choosing range (index ops) versus all.
479 * icache flushes all ways for each set, so drop icache_assoc.
480 * dcache flushes all ways and each setting of bit 12 for each
481 * index, so drop dcache_assoc and halve the dcache_sets.
482 */
483 icache_range_cutoff = icache_sets * icache_line_size;
484 dcache_range_cutoff = (dcache_sets / 2) * icache_line_size;
485
486 printk("Primary instruction cache %ldkB, %s, linesize %d bytes.\n",
487 icache_size >> 10, way_string[icache_assoc - 1],
488 icache_line_size);
489 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n",
490 dcache_size >> 10, way_string[dcache_assoc - 1],
491 dcache_line_size);
492}
493
494/*
495 * This is called from loadmmu.c. We have to set up all the
496 * memory management function pointers, as well as initialize
497 * the caches and tlbs
498 */
499void ld_mmu_sb1(void)
500{
501 extern char except_vec2_sb1;
502 extern char handle_vec2_sb1;
503
504 /* Special cache error handler for SB1 */
505 memcpy((void *)(CAC_BASE + 0x100), &except_vec2_sb1, 0x80);
506 memcpy((void *)(UNCAC_BASE + 0x100), &except_vec2_sb1, 0x80);
507 memcpy((void *)CKSEG1ADDR(&handle_vec2_sb1), &handle_vec2_sb1, 0x80);
508
509 probe_cache_sizes();
510
511#ifdef CONFIG_SIBYTE_DMA_PAGEOPS
512 sb1_dma_init();
513#endif
514
515 /*
516 * None of these are needed for the SB1 - the Dcache is
517 * physically indexed and tagged, so no virtual aliasing can
518 * occur
519 */
520 flush_cache_range = (void *) sb1_nop;
521 flush_cache_mm = (void (*)(struct mm_struct *))sb1_nop;
522 flush_cache_all = sb1_nop;
523
524 /* These routines are for Icache coherence with the Dcache */
525 flush_icache_range = sb1_flush_icache_range;
526 flush_icache_page = sb1_flush_icache_page;
527 flush_icache_all = __sb1_flush_icache_all; /* local only */
528
529 /* This implies an Icache flush too, so can't be nop'ed */
530 flush_cache_page = sb1_flush_cache_page;
531
532 flush_cache_sigtramp = sb1_flush_cache_sigtramp;
533 flush_data_cache_page = (void *) sb1_nop;
534
535 /* Full flush */
536 __flush_cache_all = sb1___flush_cache_all;
537
538 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT);
539
540 /*
541 * This is the only way to force the update of K0 to complete
542 * before subsequent instruction fetch.
543 */
544 __asm__ __volatile__(
545 ".set push \n"
546 " .set noat \n"
547 " .set noreorder \n"
548 " .set mips3 \n"
549 " " STR(PTR_LA) " $1, 1f \n"
550 " " STR(MTC0) " $1, $14 \n"
551 " eret \n"
552 "1: .set pop"
553 :
554 :
555 : "memory");
556
557 flush_cache_all();
558}
diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c
new file mode 100644
index 000000000000..ff5afab64b2f
--- /dev/null
+++ b/arch/mips/mm/c-tx39.c
@@ -0,0 +1,493 @@
1/*
2 * r2300.c: R2000 and R3000 specific mmu/cache code.
3 *
4 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
5 *
6 * with a lot of changes to make this thing work for R3000s
7 * Tx39XX R4k style caches added. HK
8 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
9 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
10 */
11#include <linux/init.h>
12#include <linux/kernel.h>
13#include <linux/sched.h>
14#include <linux/mm.h>
15
16#include <asm/cacheops.h>
17#include <asm/page.h>
18#include <asm/pgtable.h>
19#include <asm/mmu_context.h>
20#include <asm/system.h>
21#include <asm/isadep.h>
22#include <asm/io.h>
23#include <asm/bootinfo.h>
24#include <asm/cpu.h>
25
26/* For R3000 cores with R4000 style caches */
27static unsigned long icache_size, dcache_size; /* Size in bytes */
28
29#include <asm/r4kcache.h>
30
31extern int r3k_have_wired_reg; /* in r3k-tlb.c */
32
33/* This sequence is required to ensure icache is disabled immediately */
34#define TX39_STOP_STREAMING() \
35__asm__ __volatile__( \
36 ".set push\n\t" \
37 ".set noreorder\n\t" \
38 "b 1f\n\t" \
39 "nop\n\t" \
40 "1:\n\t" \
41 ".set pop" \
42 )
43
44/* TX39H-style cache flush routines. */
45static void tx39h_flush_icache_all(void)
46{
47 unsigned long start = KSEG0;
48 unsigned long end = (start + icache_size);
49 unsigned long flags, config;
50
51 /* disable icache (set ICE#) */
52 local_irq_save(flags);
53 config = read_c0_conf();
54 write_c0_conf(config & ~TX39_CONF_ICE);
55 TX39_STOP_STREAMING();
56
57 /* invalidate icache */
58 while (start < end) {
59 cache16_unroll32(start, Index_Invalidate_I);
60 start += 0x200;
61 }
62
63 write_c0_conf(config);
64 local_irq_restore(flags);
65}
66
67static void tx39h_dma_cache_wback_inv(unsigned long addr, unsigned long size)
68{
69 unsigned long end, a;
70 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
71
72 /* Catch bad driver code */
73 BUG_ON(size == 0);
74
75 iob();
76 a = addr & ~(dc_lsize - 1);
77 end = (addr + size - 1) & ~(dc_lsize - 1);
78 while (1) {
79 invalidate_dcache_line(a); /* Hit_Invalidate_D */
80 if (a == end) break;
81 a += dc_lsize;
82 }
83}
84
85
86/* TX39H2,TX39H3 */
87static inline void tx39_blast_dcache_page(unsigned long addr)
88{
89 if (current_cpu_data.cputype != CPU_TX3912)
90 blast_dcache16_page(addr);
91}
92
93static inline void tx39_blast_dcache_page_indexed(unsigned long addr)
94{
95 blast_dcache16_page_indexed(addr);
96}
97
98static inline void tx39_blast_dcache(void)
99{
100 blast_dcache16();
101}
102
103static inline void tx39_blast_icache_page(unsigned long addr)
104{
105 unsigned long flags, config;
106 /* disable icache (set ICE#) */
107 local_irq_save(flags);
108 config = read_c0_conf();
109 write_c0_conf(config & ~TX39_CONF_ICE);
110 TX39_STOP_STREAMING();
111 blast_icache16_page(addr);
112 write_c0_conf(config);
113 local_irq_restore(flags);
114}
115
116static inline void tx39_blast_icache_page_indexed(unsigned long addr)
117{
118 unsigned long flags, config;
119 /* disable icache (set ICE#) */
120 local_irq_save(flags);
121 config = read_c0_conf();
122 write_c0_conf(config & ~TX39_CONF_ICE);
123 TX39_STOP_STREAMING();
124 blast_icache16_page_indexed(addr);
125 write_c0_conf(config);
126 local_irq_restore(flags);
127}
128
129static inline void tx39_blast_icache(void)
130{
131 unsigned long flags, config;
132 /* disable icache (set ICE#) */
133 local_irq_save(flags);
134 config = read_c0_conf();
135 write_c0_conf(config & ~TX39_CONF_ICE);
136 TX39_STOP_STREAMING();
137 blast_icache16();
138 write_c0_conf(config);
139 local_irq_restore(flags);
140}
141
142static inline void tx39_flush_cache_all(void)
143{
144 if (!cpu_has_dc_aliases)
145 return;
146
147 tx39_blast_dcache();
148 tx39_blast_icache();
149}
150
151static inline void tx39___flush_cache_all(void)
152{
153 tx39_blast_dcache();
154 tx39_blast_icache();
155}
156
157static void tx39_flush_cache_mm(struct mm_struct *mm)
158{
159 if (!cpu_has_dc_aliases)
160 return;
161
162 if (cpu_context(smp_processor_id(), mm) != 0) {
163 tx39_flush_cache_all();
164 }
165}
166
167static void tx39_flush_cache_range(struct vm_area_struct *vma,
168 unsigned long start, unsigned long end)
169{
170 struct mm_struct *mm = vma->vm_mm;
171
172 if (!cpu_has_dc_aliases)
173 return;
174
175 if (cpu_context(smp_processor_id(), mm) != 0) {
176 tx39_blast_dcache();
177 tx39_blast_icache();
178 }
179}
180
181static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page, unsigned long pfn)
182{
183 int exec = vma->vm_flags & VM_EXEC;
184 struct mm_struct *mm = vma->vm_mm;
185 pgd_t *pgdp;
186 pmd_t *pmdp;
187 pte_t *ptep;
188
189 /*
190 * If ownes no valid ASID yet, cannot possibly have gotten
191 * this page into the cache.
192 */
193 if (cpu_context(smp_processor_id(), mm) == 0)
194 return;
195
196 page &= PAGE_MASK;
197 pgdp = pgd_offset(mm, page);
198 pmdp = pmd_offset(pgdp, page);
199 ptep = pte_offset(pmdp, page);
200
201 /*
202 * If the page isn't marked valid, the page cannot possibly be
203 * in the cache.
204 */
205 if (!(pte_val(*ptep) & _PAGE_PRESENT))
206 return;
207
208 /*
209 * Doing flushes for another ASID than the current one is
210 * too difficult since stupid R4k caches do a TLB translation
211 * for every cache flush operation. So we do indexed flushes
212 * in that case, which doesn't overly flush the cache too much.
213 */
214 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
215 if (cpu_has_dc_aliases || exec)
216 tx39_blast_dcache_page(page);
217 if (exec)
218 tx39_blast_icache_page(page);
219
220 return;
221 }
222
223 /*
224 * Do indexed flush, too much work to get the (possible) TLB refills
225 * to work correctly.
226 */
227 page = (KSEG0 + (page & (dcache_size - 1)));
228 if (cpu_has_dc_aliases || exec)
229 tx39_blast_dcache_page_indexed(page);
230 if (exec)
231 tx39_blast_icache_page_indexed(page);
232}
233
234static void tx39_flush_data_cache_page(unsigned long addr)
235{
236 tx39_blast_dcache_page(addr);
237}
238
239static void tx39_flush_icache_range(unsigned long start, unsigned long end)
240{
241 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
242 unsigned long addr, aend;
243
244 if (end - start > dcache_size)
245 tx39_blast_dcache();
246 else {
247 addr = start & ~(dc_lsize - 1);
248 aend = (end - 1) & ~(dc_lsize - 1);
249
250 while (1) {
251 /* Hit_Writeback_Inv_D */
252 protected_writeback_dcache_line(addr);
253 if (addr == aend)
254 break;
255 addr += dc_lsize;
256 }
257 }
258
259 if (end - start > icache_size)
260 tx39_blast_icache();
261 else {
262 unsigned long flags, config;
263 addr = start & ~(dc_lsize - 1);
264 aend = (end - 1) & ~(dc_lsize - 1);
265 /* disable icache (set ICE#) */
266 local_irq_save(flags);
267 config = read_c0_conf();
268 write_c0_conf(config & ~TX39_CONF_ICE);
269 TX39_STOP_STREAMING();
270 while (1) {
271 /* Hit_Invalidate_I */
272 protected_flush_icache_line(addr);
273 if (addr == aend)
274 break;
275 addr += dc_lsize;
276 }
277 write_c0_conf(config);
278 local_irq_restore(flags);
279 }
280}
281
282/*
283 * Ok, this seriously sucks. We use them to flush a user page but don't
284 * know the virtual address, so we have to blast away the whole icache
285 * which is significantly more expensive than the real thing. Otoh we at
286 * least know the kernel address of the page so we can flush it
287 * selectivly.
288 */
289static void tx39_flush_icache_page(struct vm_area_struct *vma, struct page *page)
290{
291 unsigned long addr;
292 /*
293 * If there's no context yet, or the page isn't executable, no icache
294 * flush is needed.
295 */
296 if (!(vma->vm_flags & VM_EXEC))
297 return;
298
299 addr = (unsigned long) page_address(page);
300 tx39_blast_dcache_page(addr);
301
302 /*
303 * We're not sure of the virtual address(es) involved here, so
304 * we have to flush the entire I-cache.
305 */
306 tx39_blast_icache();
307}
308
309static void tx39_dma_cache_wback_inv(unsigned long addr, unsigned long size)
310{
311 unsigned long end, a;
312
313 if (((size | addr) & (PAGE_SIZE - 1)) == 0) {
314 end = addr + size;
315 do {
316 tx39_blast_dcache_page(addr);
317 addr += PAGE_SIZE;
318 } while(addr != end);
319 } else if (size > dcache_size) {
320 tx39_blast_dcache();
321 } else {
322 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
323 a = addr & ~(dc_lsize - 1);
324 end = (addr + size - 1) & ~(dc_lsize - 1);
325 while (1) {
326 flush_dcache_line(a); /* Hit_Writeback_Inv_D */
327 if (a == end) break;
328 a += dc_lsize;
329 }
330 }
331}
332
333static void tx39_dma_cache_inv(unsigned long addr, unsigned long size)
334{
335 unsigned long end, a;
336
337 if (((size | addr) & (PAGE_SIZE - 1)) == 0) {
338 end = addr + size;
339 do {
340 tx39_blast_dcache_page(addr);
341 addr += PAGE_SIZE;
342 } while(addr != end);
343 } else if (size > dcache_size) {
344 tx39_blast_dcache();
345 } else {
346 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
347 a = addr & ~(dc_lsize - 1);
348 end = (addr + size - 1) & ~(dc_lsize - 1);
349 while (1) {
350 invalidate_dcache_line(a); /* Hit_Invalidate_D */
351 if (a == end) break;
352 a += dc_lsize;
353 }
354 }
355}
356
357static void tx39_flush_cache_sigtramp(unsigned long addr)
358{
359 unsigned long ic_lsize = current_cpu_data.icache.linesz;
360 unsigned long dc_lsize = current_cpu_data.dcache.linesz;
361 unsigned long config;
362 unsigned long flags;
363
364 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
365
366 /* disable icache (set ICE#) */
367 local_irq_save(flags);
368 config = read_c0_conf();
369 write_c0_conf(config & ~TX39_CONF_ICE);
370 TX39_STOP_STREAMING();
371 protected_flush_icache_line(addr & ~(ic_lsize - 1));
372 write_c0_conf(config);
373 local_irq_restore(flags);
374}
375
376static __init void tx39_probe_cache(void)
377{
378 unsigned long config;
379
380 config = read_c0_conf();
381
382 icache_size = 1 << (10 + ((config & TX39_CONF_ICS_MASK) >>
383 TX39_CONF_ICS_SHIFT));
384 dcache_size = 1 << (10 + ((config & TX39_CONF_DCS_MASK) >>
385 TX39_CONF_DCS_SHIFT));
386
387 current_cpu_data.icache.linesz = 16;
388 switch (current_cpu_data.cputype) {
389 case CPU_TX3912:
390 current_cpu_data.icache.ways = 1;
391 current_cpu_data.dcache.ways = 1;
392 current_cpu_data.dcache.linesz = 4;
393 break;
394
395 case CPU_TX3927:
396 current_cpu_data.icache.ways = 2;
397 current_cpu_data.dcache.ways = 2;
398 current_cpu_data.dcache.linesz = 16;
399 break;
400
401 case CPU_TX3922:
402 default:
403 current_cpu_data.icache.ways = 1;
404 current_cpu_data.dcache.ways = 1;
405 current_cpu_data.dcache.linesz = 16;
406 break;
407 }
408}
409
410void __init ld_mmu_tx39(void)
411{
412 extern void build_clear_page(void);
413 extern void build_copy_page(void);
414 unsigned long config;
415
416 config = read_c0_conf();
417 config &= ~TX39_CONF_WBON;
418 write_c0_conf(config);
419
420 tx39_probe_cache();
421
422 switch (current_cpu_data.cputype) {
423 case CPU_TX3912:
424 /* TX39/H core (writethru direct-map cache) */
425 flush_cache_all = tx39h_flush_icache_all;
426 __flush_cache_all = tx39h_flush_icache_all;
427 flush_cache_mm = (void *) tx39h_flush_icache_all;
428 flush_cache_range = (void *) tx39h_flush_icache_all;
429 flush_cache_page = (void *) tx39h_flush_icache_all;
430 flush_icache_page = (void *) tx39h_flush_icache_all;
431 flush_icache_range = (void *) tx39h_flush_icache_all;
432
433 flush_cache_sigtramp = (void *) tx39h_flush_icache_all;
434 flush_data_cache_page = (void *) tx39h_flush_icache_all;
435
436 _dma_cache_wback_inv = tx39h_dma_cache_wback_inv;
437
438 shm_align_mask = PAGE_SIZE - 1;
439
440 break;
441
442 case CPU_TX3922:
443 case CPU_TX3927:
444 default:
445 /* TX39/H2,H3 core (writeback 2way-set-associative cache) */
446 r3k_have_wired_reg = 1;
447 write_c0_wired(0); /* set 8 on reset... */
448 /* board-dependent init code may set WBON */
449
450 flush_cache_all = tx39_flush_cache_all;
451 __flush_cache_all = tx39___flush_cache_all;
452 flush_cache_mm = tx39_flush_cache_mm;
453 flush_cache_range = tx39_flush_cache_range;
454 flush_cache_page = tx39_flush_cache_page;
455 flush_icache_page = tx39_flush_icache_page;
456 flush_icache_range = tx39_flush_icache_range;
457
458 flush_cache_sigtramp = tx39_flush_cache_sigtramp;
459 flush_data_cache_page = tx39_flush_data_cache_page;
460
461 _dma_cache_wback_inv = tx39_dma_cache_wback_inv;
462 _dma_cache_wback = tx39_dma_cache_wback_inv;
463 _dma_cache_inv = tx39_dma_cache_inv;
464
465 shm_align_mask = max_t(unsigned long,
466 (dcache_size / current_cpu_data.dcache.ways) - 1,
467 PAGE_SIZE - 1);
468
469 break;
470 }
471
472 current_cpu_data.icache.waysize = icache_size / current_cpu_data.icache.ways;
473 current_cpu_data.dcache.waysize = dcache_size / current_cpu_data.dcache.ways;
474
475 current_cpu_data.icache.sets =
476 current_cpu_data.icache.waysize / current_cpu_data.icache.linesz;
477 current_cpu_data.dcache.sets =
478 current_cpu_data.dcache.waysize / current_cpu_data.dcache.linesz;
479
480 if (current_cpu_data.dcache.waysize > PAGE_SIZE)
481 current_cpu_data.dcache.flags |= MIPS_CACHE_ALIASES;
482
483 current_cpu_data.icache.waybit = 0;
484 current_cpu_data.dcache.waybit = 0;
485
486 printk("Primary instruction cache %ldkB, linesize %d bytes\n",
487 icache_size >> 10, current_cpu_data.icache.linesz);
488 printk("Primary data cache %ldkB, linesize %d bytes\n",
489 dcache_size >> 10, current_cpu_data.dcache.linesz);
490
491 build_clear_page();
492 build_copy_page();
493}
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
new file mode 100644
index 000000000000..1d95cdb77bed
--- /dev/null
+++ b/arch/mips/mm/cache.c
@@ -0,0 +1,157 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1994 - 2003 by Ralf Baechle
7 */
8#include <linux/config.h>
9#include <linux/init.h>
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/sched.h>
13#include <linux/mm.h>
14
15#include <asm/cacheflush.h>
16#include <asm/processor.h>
17#include <asm/cpu.h>
18#include <asm/cpu-features.h>
19
20/* Cache operations. */
21void (*flush_cache_all)(void);
22void (*__flush_cache_all)(void);
23void (*flush_cache_mm)(struct mm_struct *mm);
24void (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start,
25 unsigned long end);
26void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page, unsigned long pfn);
27void (*flush_icache_range)(unsigned long start, unsigned long end);
28void (*flush_icache_page)(struct vm_area_struct *vma, struct page *page);
29
30/* MIPS specific cache operations */
31void (*flush_cache_sigtramp)(unsigned long addr);
32void (*flush_data_cache_page)(unsigned long addr);
33void (*flush_icache_all)(void);
34
35#ifdef CONFIG_DMA_NONCOHERENT
36
37/* DMA cache operations. */
38void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size);
39void (*_dma_cache_wback)(unsigned long start, unsigned long size);
40void (*_dma_cache_inv)(unsigned long start, unsigned long size);
41
42EXPORT_SYMBOL(_dma_cache_wback_inv);
43EXPORT_SYMBOL(_dma_cache_wback);
44EXPORT_SYMBOL(_dma_cache_inv);
45
46#endif /* CONFIG_DMA_NONCOHERENT */
47
48/*
49 * We could optimize the case where the cache argument is not BCACHE but
50 * that seems very atypical use ...
51 */
52asmlinkage int sys_cacheflush(unsigned long addr, unsigned long int bytes,
53 unsigned int cache)
54{
55 if (!access_ok(VERIFY_WRITE, (void *) addr, bytes))
56 return -EFAULT;
57
58 flush_icache_range(addr, addr + bytes);
59
60 return 0;
61}
62
63void __flush_dcache_page(struct page *page)
64{
65 struct address_space *mapping = page_mapping(page);
66 unsigned long addr;
67
68 if (mapping && !mapping_mapped(mapping)) {
69 SetPageDcacheDirty(page);
70 return;
71 }
72
73 /*
74 * We could delay the flush for the !page_mapping case too. But that
75 * case is for exec env/arg pages and those are %99 certainly going to
76 * get faulted into the tlb (and thus flushed) anyways.
77 */
78 addr = (unsigned long) page_address(page);
79 flush_data_cache_page(addr);
80}
81
82EXPORT_SYMBOL(__flush_dcache_page);
83
84void __update_cache(struct vm_area_struct *vma, unsigned long address,
85 pte_t pte)
86{
87 struct page *page;
88 unsigned long pfn, addr;
89
90 pfn = pte_pfn(pte);
91 if (pfn_valid(pfn) && (page = pfn_to_page(pfn), page_mapping(page)) &&
92 Page_dcache_dirty(page)) {
93 if (pages_do_alias((unsigned long)page_address(page),
94 address & PAGE_MASK)) {
95 addr = (unsigned long) page_address(page);
96 flush_data_cache_page(addr);
97 }
98
99 ClearPageDcacheDirty(page);
100 }
101}
102
103extern void ld_mmu_r23000(void);
104extern void ld_mmu_r4xx0(void);
105extern void ld_mmu_tx39(void);
106extern void ld_mmu_r6000(void);
107extern void ld_mmu_tfp(void);
108extern void ld_mmu_andes(void);
109extern void ld_mmu_sb1(void);
110
111void __init cpu_cache_init(void)
112{
113 if (cpu_has_4ktlb) {
114#if defined(CONFIG_CPU_R4X00) || defined(CONFIG_CPU_VR41XX) || \
115 defined(CONFIG_CPU_R4300) || defined(CONFIG_CPU_R5000) || \
116 defined(CONFIG_CPU_NEVADA) || defined(CONFIG_CPU_R5432) || \
117 defined(CONFIG_CPU_R5500) || defined(CONFIG_CPU_MIPS32) || \
118 defined(CONFIG_CPU_MIPS64) || defined(CONFIG_CPU_TX49XX) || \
119 defined(CONFIG_CPU_RM7000) || defined(CONFIG_CPU_RM9000)
120 ld_mmu_r4xx0();
121#endif
122 } else switch (current_cpu_data.cputype) {
123#ifdef CONFIG_CPU_R3000
124 case CPU_R2000:
125 case CPU_R3000:
126 case CPU_R3000A:
127 case CPU_R3081E:
128 ld_mmu_r23000();
129 break;
130#endif
131#ifdef CONFIG_CPU_TX39XX
132 case CPU_TX3912:
133 case CPU_TX3922:
134 case CPU_TX3927:
135 ld_mmu_tx39();
136 break;
137#endif
138#ifdef CONFIG_CPU_R10000
139 case CPU_R10000:
140 case CPU_R12000:
141 ld_mmu_r4xx0();
142 break;
143#endif
144#ifdef CONFIG_CPU_SB1
145 case CPU_SB1:
146 ld_mmu_sb1();
147 break;
148#endif
149
150 case CPU_R8000:
151 panic("R8000 is unsupported");
152 break;
153
154 default:
155 panic("Yeee, unsupported cache architecture.");
156 }
157}
diff --git a/arch/mips/mm/cerr-sb1.c b/arch/mips/mm/cerr-sb1.c
new file mode 100644
index 000000000000..13d96d62764e
--- /dev/null
+++ b/arch/mips/mm/cerr-sb1.c
@@ -0,0 +1,543 @@
1/*
2 * Copyright (C) 2001,2002,2003 Broadcom Corporation
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 */
18#include <linux/config.h>
19#include <linux/sched.h>
20#include <asm/mipsregs.h>
21#include <asm/sibyte/sb1250.h>
22
23#ifndef CONFIG_SIBYTE_BUS_WATCHER
24#include <asm/io.h>
25#include <asm/sibyte/sb1250_regs.h>
26#include <asm/sibyte/sb1250_scd.h>
27#endif
28
29/* SB1 definitions */
30
31/* XXX should come from config1 XXX */
32#define SB1_CACHE_INDEX_MASK 0x1fe0
33
34#define CP0_ERRCTL_RECOVERABLE (1 << 31)
35#define CP0_ERRCTL_DCACHE (1 << 30)
36#define CP0_ERRCTL_ICACHE (1 << 29)
37#define CP0_ERRCTL_MULTIBUS (1 << 23)
38#define CP0_ERRCTL_MC_TLB (1 << 15)
39#define CP0_ERRCTL_MC_TIMEOUT (1 << 14)
40
41#define CP0_CERRI_TAG_PARITY (1 << 29)
42#define CP0_CERRI_DATA_PARITY (1 << 28)
43#define CP0_CERRI_EXTERNAL (1 << 26)
44
45#define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
46#define CP0_CERRI_DATA (CP0_CERRI_DATA_PARITY)
47
48#define CP0_CERRD_MULTIPLE (1 << 31)
49#define CP0_CERRD_TAG_STATE (1 << 30)
50#define CP0_CERRD_TAG_ADDRESS (1 << 29)
51#define CP0_CERRD_DATA_SBE (1 << 28)
52#define CP0_CERRD_DATA_DBE (1 << 27)
53#define CP0_CERRD_EXTERNAL (1 << 26)
54#define CP0_CERRD_LOAD (1 << 25)
55#define CP0_CERRD_STORE (1 << 24)
56#define CP0_CERRD_FILLWB (1 << 23)
57#define CP0_CERRD_COHERENCY (1 << 22)
58#define CP0_CERRD_DUPTAG (1 << 21)
59
60#define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
61#define CP0_CERRD_IDX_VALID(c) \
62 (((c) & (CP0_CERRD_LOAD | CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
63#define CP0_CERRD_CAUSES \
64 (CP0_CERRD_LOAD | CP0_CERRD_STORE | CP0_CERRD_FILLWB | CP0_CERRD_COHERENCY | CP0_CERRD_DUPTAG)
65#define CP0_CERRD_TYPES \
66 (CP0_CERRD_TAG_STATE | CP0_CERRD_TAG_ADDRESS | CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE | CP0_CERRD_EXTERNAL)
67#define CP0_CERRD_DATA (CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE)
68
69static uint32_t extract_ic(unsigned short addr, int data);
70static uint32_t extract_dc(unsigned short addr, int data);
71
72static inline void breakout_errctl(unsigned int val)
73{
74 if (val & CP0_ERRCTL_RECOVERABLE)
75 prom_printf(" recoverable");
76 if (val & CP0_ERRCTL_DCACHE)
77 prom_printf(" dcache");
78 if (val & CP0_ERRCTL_ICACHE)
79 prom_printf(" icache");
80 if (val & CP0_ERRCTL_MULTIBUS)
81 prom_printf(" multiple-buserr");
82 prom_printf("\n");
83}
84
85static inline void breakout_cerri(unsigned int val)
86{
87 if (val & CP0_CERRI_TAG_PARITY)
88 prom_printf(" tag-parity");
89 if (val & CP0_CERRI_DATA_PARITY)
90 prom_printf(" data-parity");
91 if (val & CP0_CERRI_EXTERNAL)
92 prom_printf(" external");
93 prom_printf("\n");
94}
95
96static inline void breakout_cerrd(unsigned int val)
97{
98 switch (val & CP0_CERRD_CAUSES) {
99 case CP0_CERRD_LOAD:
100 prom_printf(" load,");
101 break;
102 case CP0_CERRD_STORE:
103 prom_printf(" store,");
104 break;
105 case CP0_CERRD_FILLWB:
106 prom_printf(" fill/wb,");
107 break;
108 case CP0_CERRD_COHERENCY:
109 prom_printf(" coherency,");
110 break;
111 case CP0_CERRD_DUPTAG:
112 prom_printf(" duptags,");
113 break;
114 default:
115 prom_printf(" NO CAUSE,");
116 break;
117 }
118 if (!(val & CP0_CERRD_TYPES))
119 prom_printf(" NO TYPE");
120 else {
121 if (val & CP0_CERRD_MULTIPLE)
122 prom_printf(" multi-err");
123 if (val & CP0_CERRD_TAG_STATE)
124 prom_printf(" tag-state");
125 if (val & CP0_CERRD_TAG_ADDRESS)
126 prom_printf(" tag-address");
127 if (val & CP0_CERRD_DATA_SBE)
128 prom_printf(" data-SBE");
129 if (val & CP0_CERRD_DATA_DBE)
130 prom_printf(" data-DBE");
131 if (val & CP0_CERRD_EXTERNAL)
132 prom_printf(" external");
133 }
134 prom_printf("\n");
135}
136
137#ifndef CONFIG_SIBYTE_BUS_WATCHER
138
139static void check_bus_watcher(void)
140{
141 uint32_t status, l2_err, memio_err;
142
143 /* Destructive read, clears register and interrupt */
144 status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
145 /* Bit 31 is always on, but there's no #define for that */
146 if (status & ~(1UL << 31)) {
147 l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
148 memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
149 prom_printf("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
150 prom_printf("\nLast recorded signature:\n");
151 prom_printf("Request %02x from %d, answered by %d with Dcode %d\n",
152 (unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
153 (int)(G_SCD_BERR_TID(status) >> 6),
154 (int)G_SCD_BERR_RID(status),
155 (int)G_SCD_BERR_DCODE(status));
156 } else {
157 prom_printf("Bus watcher indicates no error\n");
158 }
159}
160#else
161extern void check_bus_watcher(void);
162#endif
163
164asmlinkage void sb1_cache_error(void)
165{
166 uint64_t cerr_dpa;
167 uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
168
169 prom_printf("Cache error exception on CPU %x:\n",
170 (read_c0_prid() >> 25) & 0x7);
171
172 __asm__ __volatile__ (
173 " .set push\n\t"
174 " .set mips64\n\t"
175 " .set noat\n\t"
176 " mfc0 %0, $26\n\t"
177 " mfc0 %1, $27\n\t"
178 " mfc0 %2, $27, 1\n\t"
179 " dmfc0 $1, $27, 3\n\t"
180 " dsrl32 %3, $1, 0 \n\t"
181 " sll %4, $1, 0 \n\t"
182 " mfc0 %5, $30\n\t"
183 " .set pop"
184 : "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
185 "=r" (dpahi), "=r" (dpalo), "=r" (eepc));
186
187 cerr_dpa = (((uint64_t)dpahi) << 32) | dpalo;
188 prom_printf(" c0_errorepc == %08x\n", eepc);
189 prom_printf(" c0_errctl == %08x", errctl);
190 breakout_errctl(errctl);
191 if (errctl & CP0_ERRCTL_ICACHE) {
192 prom_printf(" c0_cerr_i == %08x", cerr_i);
193 breakout_cerri(cerr_i);
194 if (CP0_CERRI_IDX_VALID(cerr_i)) {
195 /* Check index of EPC, allowing for delay slot */
196 if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
197 ((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
198 prom_printf(" cerr_i idx doesn't match eepc\n");
199 else {
200 res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
201 (cerr_i & CP0_CERRI_DATA) != 0);
202 if (!(res & cerr_i))
203 prom_printf("...didn't see indicated icache problem\n");
204 }
205 }
206 }
207 if (errctl & CP0_ERRCTL_DCACHE) {
208 prom_printf(" c0_cerr_d == %08x", cerr_d);
209 breakout_cerrd(cerr_d);
210 if (CP0_CERRD_DPA_VALID(cerr_d)) {
211 prom_printf(" c0_cerr_dpa == %010llx\n", cerr_dpa);
212 if (!CP0_CERRD_IDX_VALID(cerr_d)) {
213 res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
214 (cerr_d & CP0_CERRD_DATA) != 0);
215 if (!(res & cerr_d))
216 prom_printf("...didn't see indicated dcache problem\n");
217 } else {
218 if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
219 prom_printf(" cerr_d idx doesn't match cerr_dpa\n");
220 else {
221 res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
222 (cerr_d & CP0_CERRD_DATA) != 0);
223 if (!(res & cerr_d))
224 prom_printf("...didn't see indicated problem\n");
225 }
226 }
227 }
228 }
229
230 check_bus_watcher();
231
232 while (1);
233 /*
234 * This tends to make things get really ugly; let's just stall instead.
235 * panic("Can't handle the cache error!");
236 */
237}
238
239
240/* Parity lookup table. */
241static const uint8_t parity[256] = {
242 0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
243 1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
244 1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
245 0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
246 1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
247 0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
248 0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
249 1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0
250};
251
252/* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
253static const uint64_t mask_72_64[8] = {
254 0x0738C808099264FFULL,
255 0x38C808099264FF07ULL,
256 0xC808099264FF0738ULL,
257 0x08099264FF0738C8ULL,
258 0x099264FF0738C808ULL,
259 0x9264FF0738C80809ULL,
260 0x64FF0738C8080992ULL,
261 0xFF0738C808099264ULL
262};
263
264/* Calculate the parity on a range of bits */
265static char range_parity(uint64_t dword, int max, int min)
266{
267 char parity = 0;
268 int i;
269 dword >>= min;
270 for (i=max-min; i>=0; i--) {
271 if (dword & 0x1)
272 parity = !parity;
273 dword >>= 1;
274 }
275 return parity;
276}
277
278/* Calculate the 4-bit even byte-parity for an instruction */
279static unsigned char inst_parity(uint32_t word)
280{
281 int i, j;
282 char parity = 0;
283 for (j=0; j<4; j++) {
284 char byte_parity = 0;
285 for (i=0; i<8; i++) {
286 if (word & 0x80000000)
287 byte_parity = !byte_parity;
288 word <<= 1;
289 }
290 parity <<= 1;
291 parity |= byte_parity;
292 }
293 return parity;
294}
295
296static uint32_t extract_ic(unsigned short addr, int data)
297{
298 unsigned short way;
299 int valid;
300 uint64_t taglo, va, tlo_tmp;
301 uint32_t taghi, taglolo, taglohi;
302 uint8_t lru;
303 int res = 0;
304
305 prom_printf("Icache index 0x%04x ", addr);
306 for (way = 0; way < 4; way++) {
307 /* Index-load-tag-I */
308 __asm__ __volatile__ (
309 " .set push \n\t"
310 " .set noreorder \n\t"
311 " .set mips64 \n\t"
312 " .set noat \n\t"
313 " cache 4, 0(%3) \n\t"
314 " mfc0 %0, $29 \n\t"
315 " dmfc0 $1, $28 \n\t"
316 " dsrl32 %1, $1, 0 \n\t"
317 " sll %2, $1, 0 \n\t"
318 " .set pop"
319 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
320 : "r" ((way << 13) | addr));
321
322 taglo = ((unsigned long long)taglohi << 32) | taglolo;
323 if (way == 0) {
324 lru = (taghi >> 14) & 0xff;
325 prom_printf("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
326 ((addr >> 5) & 0x3), /* bank */
327 ((addr >> 7) & 0x3f), /* index */
328 (lru & 0x3),
329 ((lru >> 2) & 0x3),
330 ((lru >> 4) & 0x3),
331 ((lru >> 6) & 0x3));
332 }
333 va = (taglo & 0xC0000FFFFFFFE000ULL) | addr;
334 if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
335 va |= 0x3FFFF00000000000ULL;
336 valid = ((taghi >> 29) & 1);
337 if (valid) {
338 tlo_tmp = taglo & 0xfff3ff;
339 if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
340 prom_printf(" ** bad parity in VTag0/G/ASID\n");
341 res |= CP0_CERRI_TAG_PARITY;
342 }
343 if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
344 prom_printf(" ** bad parity in R/VTag1\n");
345 res |= CP0_CERRI_TAG_PARITY;
346 }
347 }
348 if (valid ^ ((taghi >> 27) & 1)) {
349 prom_printf(" ** bad parity for valid bit\n");
350 res |= CP0_CERRI_TAG_PARITY;
351 }
352 prom_printf(" %d [VA %016llx] [Vld? %d] raw tags: %08X-%016llX\n",
353 way, va, valid, taghi, taglo);
354
355 if (data) {
356 uint32_t datahi, insta, instb;
357 uint8_t predecode;
358 int offset;
359
360 /* (hit all banks and ways) */
361 for (offset = 0; offset < 4; offset++) {
362 /* Index-load-data-I */
363 __asm__ __volatile__ (
364 " .set push\n\t"
365 " .set noreorder\n\t"
366 " .set mips64\n\t"
367 " .set noat\n\t"
368 " cache 6, 0(%3) \n\t"
369 " mfc0 %0, $29, 1\n\t"
370 " dmfc0 $1, $28, 1\n\t"
371 " dsrl32 %1, $1, 0 \n\t"
372 " sll %2, $1, 0 \n\t"
373 " .set pop \n"
374 : "=r" (datahi), "=r" (insta), "=r" (instb)
375 : "r" ((way << 13) | addr | (offset << 3)));
376 predecode = (datahi >> 8) & 0xff;
377 if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
378 prom_printf(" ** bad parity in predecode\n");
379 res |= CP0_CERRI_DATA_PARITY;
380 }
381 /* XXXKW should/could check predecode bits themselves */
382 if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
383 prom_printf(" ** bad parity in instruction a\n");
384 res |= CP0_CERRI_DATA_PARITY;
385 }
386 if ((datahi & 0xf) ^ inst_parity(instb)) {
387 prom_printf(" ** bad parity in instruction b\n");
388 res |= CP0_CERRI_DATA_PARITY;
389 }
390 prom_printf(" %05X-%08X%08X", datahi, insta, instb);
391 }
392 prom_printf("\n");
393 }
394 }
395 return res;
396}
397
398/* Compute the ECC for a data doubleword */
399static uint8_t dc_ecc(uint64_t dword)
400{
401 uint64_t t;
402 uint32_t w;
403 uint8_t p;
404 int i;
405
406 p = 0;
407 for (i = 7; i >= 0; i--)
408 {
409 p <<= 1;
410 t = dword & mask_72_64[i];
411 w = (uint32_t)(t >> 32);
412 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
413 ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
414 w = (uint32_t)(t & 0xFFFFFFFF);
415 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
416 ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
417 }
418 return p;
419}
420
421struct dc_state {
422 unsigned char val;
423 char *name;
424};
425
426static struct dc_state dc_states[] = {
427 { 0x00, "INVALID" },
428 { 0x0f, "COH-SHD" },
429 { 0x13, "NCO-E-C" },
430 { 0x19, "NCO-E-D" },
431 { 0x16, "COH-E-C" },
432 { 0x1c, "COH-E-D" },
433 { 0xff, "*ERROR*" }
434};
435
436#define DC_TAG_VALID(state) \
437 (((state) == 0xf) || ((state) == 0x13) || ((state) == 0x19) || ((state == 0x16)) || ((state) == 0x1c))
438
439static char *dc_state_str(unsigned char state)
440{
441 struct dc_state *dsc = dc_states;
442 while (dsc->val != 0xff) {
443 if (dsc->val == state)
444 break;
445 dsc++;
446 }
447 return dsc->name;
448}
449
450static uint32_t extract_dc(unsigned short addr, int data)
451{
452 int valid, way;
453 unsigned char state;
454 uint64_t taglo, pa;
455 uint32_t taghi, taglolo, taglohi;
456 uint8_t ecc, lru;
457 int res = 0;
458
459 prom_printf("Dcache index 0x%04x ", addr);
460 for (way = 0; way < 4; way++) {
461 __asm__ __volatile__ (
462 " .set push\n\t"
463 " .set noreorder\n\t"
464 " .set mips64\n\t"
465 " .set noat\n\t"
466 " cache 5, 0(%3)\n\t" /* Index-load-tag-D */
467 " mfc0 %0, $29, 2\n\t"
468 " dmfc0 $1, $28, 2\n\t"
469 " dsrl32 %1, $1, 0\n\t"
470 " sll %2, $1, 0\n\t"
471 " .set pop"
472 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
473 : "r" ((way << 13) | addr));
474
475 taglo = ((unsigned long long)taglohi << 32) | taglolo;
476 pa = (taglo & 0xFFFFFFE000ULL) | addr;
477 if (way == 0) {
478 lru = (taghi >> 14) & 0xff;
479 prom_printf("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
480 ((addr >> 11) & 0x2) | ((addr >> 5) & 1), /* bank */
481 ((addr >> 6) & 0x3f), /* index */
482 (lru & 0x3),
483 ((lru >> 2) & 0x3),
484 ((lru >> 4) & 0x3),
485 ((lru >> 6) & 0x3));
486 }
487 state = (taghi >> 25) & 0x1f;
488 valid = DC_TAG_VALID(state);
489 prom_printf(" %d [PA %010llx] [state %s (%02x)] raw tags: %08X-%016llX\n",
490 way, pa, dc_state_str(state), state, taghi, taglo);
491 if (valid) {
492 if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
493 prom_printf(" ** bad parity in PTag1\n");
494 res |= CP0_CERRD_TAG_ADDRESS;
495 }
496 if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
497 prom_printf(" ** bad parity in PTag0\n");
498 res |= CP0_CERRD_TAG_ADDRESS;
499 }
500 } else {
501 res |= CP0_CERRD_TAG_STATE;
502 }
503
504 if (data) {
505 uint64_t datalo;
506 uint32_t datalohi, datalolo, datahi;
507 int offset;
508
509 for (offset = 0; offset < 4; offset++) {
510 /* Index-load-data-D */
511 __asm__ __volatile__ (
512 " .set push\n\t"
513 " .set noreorder\n\t"
514 " .set mips64\n\t"
515 " .set noat\n\t"
516 " cache 7, 0(%3)\n\t" /* Index-load-data-D */
517 " mfc0 %0, $29, 3\n\t"
518 " dmfc0 $1, $28, 3\n\t"
519 " dsrl32 %1, $1, 0 \n\t"
520 " sll %2, $1, 0 \n\t"
521 " .set pop"
522 : "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
523 : "r" ((way << 13) | addr | (offset << 3)));
524 datalo = ((unsigned long long)datalohi << 32) | datalolo;
525 ecc = dc_ecc(datalo);
526 if (ecc != datahi) {
527 int bits = 0;
528 prom_printf(" ** bad ECC (%02x %02x) ->",
529 datahi, ecc);
530 ecc ^= datahi;
531 while (ecc) {
532 if (ecc & 1) bits++;
533 ecc >>= 1;
534 }
535 res |= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
536 }
537 prom_printf(" %02X-%016llX", datahi, datalo);
538 }
539 prom_printf("\n");
540 }
541 }
542 return res;
543}
diff --git a/arch/mips/mm/cex-gen.S b/arch/mips/mm/cex-gen.S
new file mode 100644
index 000000000000..e743622fd24d
--- /dev/null
+++ b/arch/mips/mm/cex-gen.S
@@ -0,0 +1,42 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1995 - 1999 Ralf Baechle
7 * Copyright (C) 1999 Silicon Graphics, Inc.
8 *
9 * Cache error handler
10 */
11#include <asm/asm.h>
12#include <asm/regdef.h>
13#include <asm/mipsregs.h>
14#include <asm/stackframe.h>
15
16/*
17 * Game over. Go to the button. Press gently. Swear where allowed by
18 * legislation.
19 */
20 LEAF(except_vec2_generic)
21 .set noreorder
22 .set noat
23 .set mips0
24 /*
25 * This is a very bad place to be. Our cache error
26 * detection has triggered. If we have write-back data
27 * in the cache, we may not be able to recover. As a
28 * first-order desperate measure, turn off KSEG0 cacheing.
29 */
30 mfc0 k0,CP0_CONFIG
31 li k1,~CONF_CM_CMASK
32 and k0,k0,k1
33 ori k0,k0,CONF_CM_UNCACHED
34 mtc0 k0,CP0_CONFIG
35 /* Give it a few cycles to sink in... */
36 nop
37 nop
38 nop
39
40 j cache_parity_error
41 nop
42 END(except_vec2_generic)
diff --git a/arch/mips/mm/cex-sb1.S b/arch/mips/mm/cex-sb1.S
new file mode 100644
index 000000000000..2c3a23aa88c3
--- /dev/null
+++ b/arch/mips/mm/cex-sb1.S
@@ -0,0 +1,170 @@
1/*
2 * Copyright (C) 2001,2002,2003 Broadcom Corporation
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 */
18#include <linux/init.h>
19
20#include <asm/asm.h>
21#include <asm/regdef.h>
22#include <asm/mipsregs.h>
23#include <asm/stackframe.h>
24#include <asm/cacheops.h>
25#include <asm/sibyte/board.h>
26
27#define C0_ERRCTL $26 /* CP0: Error info */
28#define C0_CERR_I $27 /* CP0: Icache error */
29#define C0_CERR_D $27,1 /* CP0: Dcache error */
30
31 /*
32 * Based on SiByte sample software cache-err/cerr.S
33 * CVS revision 1.8. Only the 'unrecoverable' case
34 * is changed.
35 */
36
37 __INIT
38
39 .set mips64
40 .set noreorder
41 .set noat
42
43 /*
44 * sb1_cerr_vec: code to be copied to the Cache Error
45 * Exception vector. The code must be pushed out to memory
46 * (either by copying to Kseg0 and Kseg1 both, or by flushing
47 * the L1 and L2) since it is fetched as 0xa0000100.
48 *
49 * NOTE: Be sure this handler is at most 28 instructions long
50 * since the final 16 bytes of the exception vector memory
51 * (0x170-0x17f) are used to preserve k0, k1, and ra.
52 */
53
54LEAF(except_vec2_sb1)
55 /*
56 * If this error is recoverable, we need to exit the handler
57 * without having dirtied any registers. To do this,
58 * save/restore k0 and k1 from low memory (Useg is direct
59 * mapped while ERL=1). Note that we can't save to a
60 * CPU-specific location without ruining a register in the
61 * process. This means we are vulnerable to data corruption
62 * whenever the handler is reentered by a second CPU.
63 */
64 sd k0,0x170($0)
65 sd k1,0x178($0)
66
67 /*
68 * M_ERRCTL_RECOVERABLE is bit 31, which makes it easy to tell
69 * if we can fast-path out of here for a h/w-recovered error.
70 */
71 mfc0 k1,C0_ERRCTL
72 bgtz k1,attempt_recovery
73 sll k0,k1,1
74
75recovered_dcache:
76 /*
77 * Unlock CacheErr-D (which in turn unlocks CacheErr-DPA).
78 * Ought to log the occurence of this recovered dcache error.
79 */
80 b recovered
81 mtc0 $0,C0_CERR_D
82
83attempt_recovery:
84 /*
85 * k0 has C0_ERRCTL << 1, which puts 'DC' at bit 31. Any
86 * Dcache errors we can recover from will take more extensive
87 * processing. For now, they are considered "unrecoverable".
88 * Note that 'DC' becoming set (outside of ERL mode) will
89 * cause 'IC' to clear; so if there's an Icache error, we'll
90 * only find out about it if we recover from this error and
91 * continue executing.
92 */
93 bltz k0,unrecoverable
94 sll k0,1
95
96 /*
97 * k0 has C0_ERRCTL << 2, which puts 'IC' at bit 31. If an
98 * Icache error isn't indicated, I'm not sure why we got here.
99 * Consider that case "unrecoverable" for now.
100 */
101 bgez k0,unrecoverable
102
103attempt_icache_recovery:
104 /*
105 * External icache errors are due to uncorrectable ECC errors
106 * in the L2 cache or Memory Controller and cannot be
107 * recovered here.
108 */
109 mfc0 k0,C0_CERR_I /* delay slot */
110 li k1,1 << 26 /* ICACHE_EXTERNAL */
111 and k1,k0
112 bnez k1,unrecoverable
113 andi k0,0x1fe0
114
115 /*
116 * Since the error is internal, the 'IDX' field from
117 * CacheErr-I is valid and we can just invalidate all blocks
118 * in that set.
119 */
120 cache Index_Invalidate_I,(0<<13)(k0)
121 cache Index_Invalidate_I,(1<<13)(k0)
122 cache Index_Invalidate_I,(2<<13)(k0)
123 cache Index_Invalidate_I,(3<<13)(k0)
124
125 /* Ought to log this recovered icache error */
126
127recovered:
128 /* Restore the saved registers */
129 ld k0,0x170($0)
130 ld k1,0x178($0)
131 eret
132
133unrecoverable:
134 /* Unrecoverable Icache or Dcache error; log it and/or fail */
135 j handle_vec2_sb1
136 nop
137
138END(except_vec2_sb1)
139
140 __FINIT
141
142 LEAF(handle_vec2_sb1)
143 mfc0 k0,CP0_CONFIG
144 li k1,~CONF_CM_CMASK
145 and k0,k0,k1
146 ori k0,k0,CONF_CM_UNCACHED
147 mtc0 k0,CP0_CONFIG
148
149 SSNOP
150 SSNOP
151 SSNOP
152 SSNOP
153 bnezl $0, 1f
1541:
155 mfc0 k0, CP0_STATUS
156 sll k0, k0, 3 # check CU0 (kernel?)
157 bltz k0, 2f
158 nop
159
160 /* Get a valid Kseg0 stack pointer. Any task's stack pointer
161 * will do, although if we ever want to resume execution we
162 * better not have corrupted any state. */
163 get_saved_sp
164 move sp, k1
165
1662:
167 j sb1_cache_error
168 nop
169
170 END(handle_vec2_sb1)
diff --git a/arch/mips/mm/dma-coherent.c b/arch/mips/mm/dma-coherent.c
new file mode 100644
index 000000000000..97a50d38c98f
--- /dev/null
+++ b/arch/mips/mm/dma-coherent.c
@@ -0,0 +1,255 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10#include <linux/config.h>
11#include <linux/types.h>
12#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/string.h>
15#include <linux/pci.h>
16
17#include <asm/cache.h>
18#include <asm/io.h>
19
20void *dma_alloc_noncoherent(struct device *dev, size_t size,
21 dma_addr_t * dma_handle, int gfp)
22{
23 void *ret;
24 /* ignore region specifiers */
25 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
26
27 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
28 gfp |= GFP_DMA;
29 ret = (void *) __get_free_pages(gfp, get_order(size));
30
31 if (ret != NULL) {
32 memset(ret, 0, size);
33 *dma_handle = virt_to_phys(ret);
34 }
35
36 return ret;
37}
38
39EXPORT_SYMBOL(dma_alloc_noncoherent);
40
41void *dma_alloc_coherent(struct device *dev, size_t size,
42 dma_addr_t * dma_handle, int gfp)
43 __attribute__((alias("dma_alloc_noncoherent")));
44
45EXPORT_SYMBOL(dma_alloc_coherent);
46
47void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
48 dma_addr_t dma_handle)
49{
50 unsigned long addr = (unsigned long) vaddr;
51
52 free_pages(addr, get_order(size));
53}
54
55EXPORT_SYMBOL(dma_free_noncoherent);
56
57void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
58 dma_addr_t dma_handle) __attribute__((alias("dma_free_noncoherent")));
59
60EXPORT_SYMBOL(dma_free_coherent);
61
62dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
63 enum dma_data_direction direction)
64{
65 BUG_ON(direction == DMA_NONE);
66
67 return __pa(ptr);
68}
69
70EXPORT_SYMBOL(dma_map_single);
71
72void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
73 enum dma_data_direction direction)
74{
75 BUG_ON(direction == DMA_NONE);
76}
77
78EXPORT_SYMBOL(dma_unmap_single);
79
80int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
81 enum dma_data_direction direction)
82{
83 int i;
84
85 BUG_ON(direction == DMA_NONE);
86
87 for (i = 0; i < nents; i++, sg++) {
88 sg->dma_address = (dma_addr_t)page_to_phys(sg->page) + sg->offset;
89 }
90
91 return nents;
92}
93
94EXPORT_SYMBOL(dma_map_sg);
95
96dma_addr_t dma_map_page(struct device *dev, struct page *page,
97 unsigned long offset, size_t size, enum dma_data_direction direction)
98{
99 BUG_ON(direction == DMA_NONE);
100
101 return page_to_phys(page) + offset;
102}
103
104EXPORT_SYMBOL(dma_map_page);
105
106void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
107 enum dma_data_direction direction)
108{
109 BUG_ON(direction == DMA_NONE);
110}
111
112EXPORT_SYMBOL(dma_unmap_page);
113
114void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
115 enum dma_data_direction direction)
116{
117 BUG_ON(direction == DMA_NONE);
118}
119
120EXPORT_SYMBOL(dma_unmap_sg);
121
122void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
123 size_t size, enum dma_data_direction direction)
124{
125 BUG_ON(direction == DMA_NONE);
126}
127
128EXPORT_SYMBOL(dma_sync_single_for_cpu);
129
130void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
131 size_t size, enum dma_data_direction direction)
132{
133 BUG_ON(direction == DMA_NONE);
134}
135
136EXPORT_SYMBOL(dma_sync_single_for_device);
137
138void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
139 unsigned long offset, size_t size,
140 enum dma_data_direction direction)
141{
142 BUG_ON(direction == DMA_NONE);
143}
144
145EXPORT_SYMBOL(dma_sync_single_range_for_cpu);
146
147void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle,
148 unsigned long offset, size_t size,
149 enum dma_data_direction direction)
150{
151 BUG_ON(direction == DMA_NONE);
152}
153
154EXPORT_SYMBOL(dma_sync_single_range_for_device);
155
156void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
157 enum dma_data_direction direction)
158{
159 BUG_ON(direction == DMA_NONE);
160}
161
162EXPORT_SYMBOL(dma_sync_sg_for_cpu);
163
164void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
165 enum dma_data_direction direction)
166{
167 BUG_ON(direction == DMA_NONE);
168}
169
170EXPORT_SYMBOL(dma_sync_sg_for_device);
171
172int dma_mapping_error(dma_addr_t dma_addr)
173{
174 return 0;
175}
176
177EXPORT_SYMBOL(dma_mapping_error);
178
179int dma_supported(struct device *dev, u64 mask)
180{
181 /*
182 * we fall back to GFP_DMA when the mask isn't all 1s,
183 * so we can't guarantee allocations that must be
184 * within a tighter range than GFP_DMA..
185 */
186 if (mask < 0x00ffffff)
187 return 0;
188
189 return 1;
190}
191
192EXPORT_SYMBOL(dma_supported);
193
194int dma_is_consistent(dma_addr_t dma_addr)
195{
196 return 1;
197}
198
199EXPORT_SYMBOL(dma_is_consistent);
200
201void dma_cache_sync(void *vaddr, size_t size,
202 enum dma_data_direction direction)
203{
204 BUG_ON(direction == DMA_NONE);
205}
206
207EXPORT_SYMBOL(dma_cache_sync);
208
209/* The DAC routines are a PCIism.. */
210
211#ifdef CONFIG_PCI
212
213#include <linux/pci.h>
214
215dma64_addr_t pci_dac_page_to_dma(struct pci_dev *pdev,
216 struct page *page, unsigned long offset, int direction)
217{
218 return (dma64_addr_t)page_to_phys(page) + offset;
219}
220
221EXPORT_SYMBOL(pci_dac_page_to_dma);
222
223struct page *pci_dac_dma_to_page(struct pci_dev *pdev,
224 dma64_addr_t dma_addr)
225{
226 return mem_map + (dma_addr >> PAGE_SHIFT);
227}
228
229EXPORT_SYMBOL(pci_dac_dma_to_page);
230
231unsigned long pci_dac_dma_to_offset(struct pci_dev *pdev,
232 dma64_addr_t dma_addr)
233{
234 return dma_addr & ~PAGE_MASK;
235}
236
237EXPORT_SYMBOL(pci_dac_dma_to_offset);
238
239void pci_dac_dma_sync_single_for_cpu(struct pci_dev *pdev,
240 dma64_addr_t dma_addr, size_t len, int direction)
241{
242 BUG_ON(direction == PCI_DMA_NONE);
243}
244
245EXPORT_SYMBOL(pci_dac_dma_sync_single_for_cpu);
246
247void pci_dac_dma_sync_single_for_device(struct pci_dev *pdev,
248 dma64_addr_t dma_addr, size_t len, int direction)
249{
250 BUG_ON(direction == PCI_DMA_NONE);
251}
252
253EXPORT_SYMBOL(pci_dac_dma_sync_single_for_device);
254
255#endif /* CONFIG_PCI */
diff --git a/arch/mips/mm/dma-ip27.c b/arch/mips/mm/dma-ip27.c
new file mode 100644
index 000000000000..aa7c94b5d781
--- /dev/null
+++ b/arch/mips/mm/dma-ip27.c
@@ -0,0 +1,257 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10#include <linux/types.h>
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/string.h>
14#include <linux/pci.h>
15
16#include <asm/cache.h>
17#include <asm/pci/bridge.h>
18
19#define pdev_to_baddr(pdev, addr) \
20 (BRIDGE_CONTROLLER(pdev->bus)->baddr + (addr))
21#define dev_to_baddr(dev, addr) \
22 pdev_to_baddr(to_pci_dev(dev), (addr))
23
24void *dma_alloc_noncoherent(struct device *dev, size_t size,
25 dma_addr_t * dma_handle, int gfp)
26{
27 void *ret;
28
29 /* ignore region specifiers */
30 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
31
32 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
33 gfp |= GFP_DMA;
34 ret = (void *) __get_free_pages(gfp, get_order(size));
35
36 if (ret != NULL) {
37 memset(ret, 0, size);
38 *dma_handle = dev_to_baddr(dev, virt_to_phys(ret));
39 }
40
41 return ret;
42}
43
44EXPORT_SYMBOL(dma_alloc_noncoherent);
45
46void *dma_alloc_coherent(struct device *dev, size_t size,
47 dma_addr_t * dma_handle, int gfp)
48 __attribute__((alias("dma_alloc_noncoherent")));
49
50EXPORT_SYMBOL(dma_alloc_coherent);
51
52void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
53 dma_addr_t dma_handle)
54{
55 unsigned long addr = (unsigned long) vaddr;
56
57 free_pages(addr, get_order(size));
58}
59
60EXPORT_SYMBOL(dma_free_noncoherent);
61
62void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
63 dma_addr_t dma_handle) __attribute__((alias("dma_free_noncoherent")));
64
65EXPORT_SYMBOL(dma_free_coherent);
66
67dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
68 enum dma_data_direction direction)
69{
70 BUG_ON(direction == DMA_NONE);
71
72 return dev_to_baddr(dev, __pa(ptr));
73}
74
75EXPORT_SYMBOL(dma_map_single);
76
77void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
78 enum dma_data_direction direction)
79{
80 BUG_ON(direction == DMA_NONE);
81}
82
83EXPORT_SYMBOL(dma_unmap_single);
84
85int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
86 enum dma_data_direction direction)
87{
88 int i;
89
90 BUG_ON(direction == DMA_NONE);
91
92 for (i = 0; i < nents; i++, sg++) {
93 sg->dma_address = (dma_addr_t) dev_to_baddr(dev,
94 page_to_phys(sg->page) + sg->offset);
95 }
96
97 return nents;
98}
99
100EXPORT_SYMBOL(dma_map_sg);
101
102dma_addr_t dma_map_page(struct device *dev, struct page *page,
103 unsigned long offset, size_t size, enum dma_data_direction direction)
104{
105 BUG_ON(direction == DMA_NONE);
106
107 return dev_to_baddr(dev, page_to_phys(page) + offset);
108}
109
110EXPORT_SYMBOL(dma_map_page);
111
112void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
113 enum dma_data_direction direction)
114{
115 BUG_ON(direction == DMA_NONE);
116}
117
118EXPORT_SYMBOL(dma_unmap_page);
119
120void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
121 enum dma_data_direction direction)
122{
123 BUG_ON(direction == DMA_NONE);
124}
125
126EXPORT_SYMBOL(dma_unmap_sg);
127
128void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
129 enum dma_data_direction direction)
130{
131 BUG_ON(direction == DMA_NONE);
132}
133
134EXPORT_SYMBOL(dma_sync_single_for_cpu);
135
136void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size,
137 enum dma_data_direction direction)
138{
139 BUG_ON(direction == DMA_NONE);
140}
141
142EXPORT_SYMBOL(dma_sync_single_for_device);
143
144void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
145 unsigned long offset, size_t size,
146 enum dma_data_direction direction)
147{
148 BUG_ON(direction == DMA_NONE);
149}
150
151EXPORT_SYMBOL(dma_sync_single_range_for_cpu);
152
153void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle,
154 unsigned long offset, size_t size,
155 enum dma_data_direction direction)
156{
157 BUG_ON(direction == DMA_NONE);
158}
159
160EXPORT_SYMBOL(dma_sync_single_range_for_device);
161
162void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
163 enum dma_data_direction direction)
164{
165 BUG_ON(direction == DMA_NONE);
166}
167
168EXPORT_SYMBOL(dma_sync_sg_for_cpu);
169
170void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
171 enum dma_data_direction direction)
172{
173 BUG_ON(direction == DMA_NONE);
174}
175
176EXPORT_SYMBOL(dma_sync_sg_for_device);
177
178int dma_mapping_error(dma_addr_t dma_addr)
179{
180 return 0;
181}
182
183EXPORT_SYMBOL(dma_mapping_error);
184
185int dma_supported(struct device *dev, u64 mask)
186{
187 /*
188 * we fall back to GFP_DMA when the mask isn't all 1s,
189 * so we can't guarantee allocations that must be
190 * within a tighter range than GFP_DMA..
191 */
192 if (mask < 0x00ffffff)
193 return 0;
194
195 return 1;
196}
197
198EXPORT_SYMBOL(dma_supported);
199
200int dma_is_consistent(dma_addr_t dma_addr)
201{
202 return 1;
203}
204
205EXPORT_SYMBOL(dma_is_consistent);
206
207void dma_cache_sync(void *vaddr, size_t size,
208 enum dma_data_direction direction)
209{
210 BUG_ON(direction == DMA_NONE);
211}
212
213EXPORT_SYMBOL(dma_cache_sync);
214
215dma64_addr_t pci_dac_page_to_dma(struct pci_dev *pdev,
216 struct page *page, unsigned long offset, int direction)
217{
218 dma64_addr_t addr = page_to_phys(page) + offset;
219
220 return (dma64_addr_t) pdev_to_baddr(pdev, addr);
221}
222
223EXPORT_SYMBOL(pci_dac_page_to_dma);
224
225struct page *pci_dac_dma_to_page(struct pci_dev *pdev,
226 dma64_addr_t dma_addr)
227{
228 struct bridge_controller *bc = BRIDGE_CONTROLLER(pdev->bus);
229
230 return pfn_to_page((dma_addr - bc->baddr) >> PAGE_SHIFT);
231}
232
233EXPORT_SYMBOL(pci_dac_dma_to_page);
234
235unsigned long pci_dac_dma_to_offset(struct pci_dev *pdev,
236 dma64_addr_t dma_addr)
237{
238 return dma_addr & ~PAGE_MASK;
239}
240
241EXPORT_SYMBOL(pci_dac_dma_to_offset);
242
243void pci_dac_dma_sync_single_for_cpu(struct pci_dev *pdev,
244 dma64_addr_t dma_addr, size_t len, int direction)
245{
246 BUG_ON(direction == PCI_DMA_NONE);
247}
248
249EXPORT_SYMBOL(pci_dac_dma_sync_single_for_cpu);
250
251void pci_dac_dma_sync_single_for_device(struct pci_dev *pdev,
252 dma64_addr_t dma_addr, size_t len, int direction)
253{
254 BUG_ON(direction == PCI_DMA_NONE);
255}
256
257EXPORT_SYMBOL(pci_dac_dma_sync_single_for_device);
diff --git a/arch/mips/mm/dma-ip32.c b/arch/mips/mm/dma-ip32.c
new file mode 100644
index 000000000000..2cbe196c35fb
--- /dev/null
+++ b/arch/mips/mm/dma-ip32.c
@@ -0,0 +1,382 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
8 * Copyright (C) 2005 Ilya A. Volynets-Evenbakh <ilya@total-knowledge.com>
9 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
10 * IP32 changes by Ilya.
11 */
12#include <linux/types.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/string.h>
16#include <linux/dma-mapping.h>
17
18#include <asm/cache.h>
19#include <asm/io.h>
20#include <asm/ip32/crime.h>
21
22/*
23 * Warning on the terminology - Linux calls an uncached area coherent;
24 * MIPS terminology calls memory areas with hardware maintained coherency
25 * coherent.
26 */
27
28/*
29 * Few notes.
30 * 1. CPU sees memory as two chunks: 0-256M@0x0, and the rest @0x40000000+256M
31 * 2. PCI sees memory as one big chunk @0x0 (or we could use 0x40000000 for native-endian)
32 * 3. All other devices see memory as one big chunk at 0x40000000
33 * 4. Non-PCI devices will pass NULL as struct device*
34 * Thus we translate differently, depending on device.
35 */
36
37#define RAM_OFFSET_MASK 0x3fffffff
38
39void *dma_alloc_noncoherent(struct device *dev, size_t size,
40 dma_addr_t * dma_handle, int gfp)
41{
42 void *ret;
43 /* ignore region specifiers */
44 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
45
46 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
47 gfp |= GFP_DMA;
48 ret = (void *) __get_free_pages(gfp, get_order(size));
49
50 if (ret != NULL) {
51 unsigned long addr = virt_to_phys(ret)&RAM_OFFSET_MASK;
52 memset(ret, 0, size);
53 if(dev==NULL)
54 addr+= CRIME_HI_MEM_BASE;
55 *dma_handle = addr;
56 }
57
58 return ret;
59}
60
61EXPORT_SYMBOL(dma_alloc_noncoherent);
62
63void *dma_alloc_coherent(struct device *dev, size_t size,
64 dma_addr_t * dma_handle, int gfp)
65{
66 void *ret;
67
68 ret = dma_alloc_noncoherent(dev, size, dma_handle, gfp);
69 if (ret) {
70 dma_cache_wback_inv((unsigned long) ret, size);
71 ret = UNCAC_ADDR(ret);
72 }
73
74 return ret;
75}
76
77EXPORT_SYMBOL(dma_alloc_coherent);
78
79void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
80 dma_addr_t dma_handle)
81{
82 free_pages((unsigned long) vaddr, get_order(size));
83}
84
85EXPORT_SYMBOL(dma_free_noncoherent);
86
87void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
88 dma_addr_t dma_handle)
89{
90 unsigned long addr = (unsigned long) vaddr;
91
92 addr = CAC_ADDR(addr);
93 free_pages(addr, get_order(size));
94}
95
96EXPORT_SYMBOL(dma_free_coherent);
97
98static inline void __dma_sync(unsigned long addr, size_t size,
99 enum dma_data_direction direction)
100{
101 switch (direction) {
102 case DMA_TO_DEVICE:
103 dma_cache_wback(addr, size);
104 break;
105
106 case DMA_FROM_DEVICE:
107 dma_cache_inv(addr, size);
108 break;
109
110 case DMA_BIDIRECTIONAL:
111 dma_cache_wback_inv(addr, size);
112 break;
113
114 default:
115 BUG();
116 }
117}
118
119dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
120 enum dma_data_direction direction)
121{
122 unsigned long addr = (unsigned long) ptr;
123
124 switch (direction) {
125 case DMA_TO_DEVICE:
126 dma_cache_wback(addr, size);
127 break;
128
129 case DMA_FROM_DEVICE:
130 dma_cache_inv(addr, size);
131 break;
132
133 case DMA_BIDIRECTIONAL:
134 dma_cache_wback_inv(addr, size);
135 break;
136
137 default:
138 BUG();
139 }
140
141 addr = virt_to_phys(ptr)&RAM_OFFSET_MASK;;
142 if(dev == NULL)
143 addr+=CRIME_HI_MEM_BASE;
144 return (dma_addr_t)addr;
145}
146
147EXPORT_SYMBOL(dma_map_single);
148
149void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
150 enum dma_data_direction direction)
151{
152 switch (direction) {
153 case DMA_TO_DEVICE:
154 break;
155
156 case DMA_FROM_DEVICE:
157 break;
158
159 case DMA_BIDIRECTIONAL:
160 break;
161
162 default:
163 BUG();
164 }
165}
166
167EXPORT_SYMBOL(dma_unmap_single);
168
169int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
170 enum dma_data_direction direction)
171{
172 int i;
173
174 BUG_ON(direction == DMA_NONE);
175
176 for (i = 0; i < nents; i++, sg++) {
177 unsigned long addr;
178
179 addr = (unsigned long) page_address(sg->page)+sg->offset;
180 if (addr)
181 __dma_sync(addr, sg->length, direction);
182 addr = __pa(addr)&RAM_OFFSET_MASK;;
183 if(dev == NULL)
184 addr += CRIME_HI_MEM_BASE;
185 sg->dma_address = (dma_addr_t)addr;
186 }
187
188 return nents;
189}
190
191EXPORT_SYMBOL(dma_map_sg);
192
193dma_addr_t dma_map_page(struct device *dev, struct page *page,
194 unsigned long offset, size_t size, enum dma_data_direction direction)
195{
196 unsigned long addr;
197
198 BUG_ON(direction == DMA_NONE);
199
200 addr = (unsigned long) page_address(page) + offset;
201 dma_cache_wback_inv(addr, size);
202 addr = __pa(addr)&RAM_OFFSET_MASK;;
203 if(dev == NULL)
204 addr += CRIME_HI_MEM_BASE;
205
206 return (dma_addr_t)addr;
207}
208
209EXPORT_SYMBOL(dma_map_page);
210
211void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
212 enum dma_data_direction direction)
213{
214 BUG_ON(direction == DMA_NONE);
215
216 if (direction != DMA_TO_DEVICE) {
217 unsigned long addr;
218
219 dma_address&=RAM_OFFSET_MASK;
220 addr = dma_address + PAGE_OFFSET;
221 if(dma_address>=256*1024*1024)
222 addr+=CRIME_HI_MEM_BASE;
223 dma_cache_wback_inv(addr, size);
224 }
225}
226
227EXPORT_SYMBOL(dma_unmap_page);
228
229void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
230 enum dma_data_direction direction)
231{
232 unsigned long addr;
233 int i;
234
235 BUG_ON(direction == DMA_NONE);
236
237 if (direction == DMA_TO_DEVICE)
238 return;
239
240 for (i = 0; i < nhwentries; i++, sg++) {
241 addr = (unsigned long) page_address(sg->page);
242 if (!addr)
243 continue;
244 dma_cache_wback_inv(addr + sg->offset, sg->length);
245 }
246}
247
248EXPORT_SYMBOL(dma_unmap_sg);
249
250void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
251 size_t size, enum dma_data_direction direction)
252{
253 unsigned long addr;
254
255 BUG_ON(direction == DMA_NONE);
256
257 dma_handle&=RAM_OFFSET_MASK;
258 addr = dma_handle + PAGE_OFFSET;
259 if(dma_handle>=256*1024*1024)
260 addr+=CRIME_HI_MEM_BASE;
261 __dma_sync(addr, size, direction);
262}
263
264EXPORT_SYMBOL(dma_sync_single_for_cpu);
265
266void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
267 size_t size, enum dma_data_direction direction)
268{
269 unsigned long addr;
270
271 BUG_ON(direction == DMA_NONE);
272
273 dma_handle&=RAM_OFFSET_MASK;
274 addr = dma_handle + PAGE_OFFSET;
275 if(dma_handle>=256*1024*1024)
276 addr+=CRIME_HI_MEM_BASE;
277 __dma_sync(addr, size, direction);
278}
279
280EXPORT_SYMBOL(dma_sync_single_for_device);
281
282void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
283 unsigned long offset, size_t size, enum dma_data_direction direction)
284{
285 unsigned long addr;
286
287 BUG_ON(direction == DMA_NONE);
288
289 dma_handle&=RAM_OFFSET_MASK;
290 addr = dma_handle + offset + PAGE_OFFSET;
291 if(dma_handle>=256*1024*1024)
292 addr+=CRIME_HI_MEM_BASE;
293 __dma_sync(addr, size, direction);
294}
295
296EXPORT_SYMBOL(dma_sync_single_range_for_cpu);
297
298void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle,
299 unsigned long offset, size_t size, enum dma_data_direction direction)
300{
301 unsigned long addr;
302
303 BUG_ON(direction == DMA_NONE);
304
305 dma_handle&=RAM_OFFSET_MASK;
306 addr = dma_handle + offset + PAGE_OFFSET;
307 if(dma_handle>=256*1024*1024)
308 addr+=CRIME_HI_MEM_BASE;
309 __dma_sync(addr, size, direction);
310}
311
312EXPORT_SYMBOL(dma_sync_single_range_for_device);
313
314void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
315 enum dma_data_direction direction)
316{
317 int i;
318
319 BUG_ON(direction == DMA_NONE);
320
321 /* Make sure that gcc doesn't leave the empty loop body. */
322 for (i = 0; i < nelems; i++, sg++)
323 __dma_sync((unsigned long)page_address(sg->page),
324 sg->length, direction);
325}
326
327EXPORT_SYMBOL(dma_sync_sg_for_cpu);
328
329void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
330 enum dma_data_direction direction)
331{
332 int i;
333
334 BUG_ON(direction == DMA_NONE);
335
336 /* Make sure that gcc doesn't leave the empty loop body. */
337 for (i = 0; i < nelems; i++, sg++)
338 __dma_sync((unsigned long)page_address(sg->page),
339 sg->length, direction);
340}
341
342EXPORT_SYMBOL(dma_sync_sg_for_device);
343
344int dma_mapping_error(dma_addr_t dma_addr)
345{
346 return 0;
347}
348
349EXPORT_SYMBOL(dma_mapping_error);
350
351int dma_supported(struct device *dev, u64 mask)
352{
353 /*
354 * we fall back to GFP_DMA when the mask isn't all 1s,
355 * so we can't guarantee allocations that must be
356 * within a tighter range than GFP_DMA..
357 */
358 if (mask < 0x00ffffff)
359 return 0;
360
361 return 1;
362}
363
364EXPORT_SYMBOL(dma_supported);
365
366int dma_is_consistent(dma_addr_t dma_addr)
367{
368 return 1;
369}
370
371EXPORT_SYMBOL(dma_is_consistent);
372
373void dma_cache_sync(void *vaddr, size_t size, enum dma_data_direction direction)
374{
375 if (direction == DMA_NONE)
376 return;
377
378 dma_cache_wback_inv((unsigned long)vaddr, size);
379}
380
381EXPORT_SYMBOL(dma_cache_sync);
382
diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c
new file mode 100644
index 000000000000..9895e32b0fce
--- /dev/null
+++ b/arch/mips/mm/dma-noncoherent.c
@@ -0,0 +1,400 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10#include <linux/config.h>
11#include <linux/types.h>
12#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/string.h>
15#include <linux/dma-mapping.h>
16
17#include <asm/cache.h>
18#include <asm/io.h>
19
20/*
21 * Warning on the terminology - Linux calls an uncached area coherent;
22 * MIPS terminology calls memory areas with hardware maintained coherency
23 * coherent.
24 */
25
26void *dma_alloc_noncoherent(struct device *dev, size_t size,
27 dma_addr_t * dma_handle, int gfp)
28{
29 void *ret;
30 /* ignore region specifiers */
31 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
32
33 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
34 gfp |= GFP_DMA;
35 ret = (void *) __get_free_pages(gfp, get_order(size));
36
37 if (ret != NULL) {
38 memset(ret, 0, size);
39 *dma_handle = virt_to_phys(ret);
40 }
41
42 return ret;
43}
44
45EXPORT_SYMBOL(dma_alloc_noncoherent);
46
47void *dma_alloc_coherent(struct device *dev, size_t size,
48 dma_addr_t * dma_handle, int gfp)
49{
50 void *ret;
51
52 ret = dma_alloc_noncoherent(dev, size, dma_handle, gfp);
53 if (ret) {
54 dma_cache_wback_inv((unsigned long) ret, size);
55 ret = UNCAC_ADDR(ret);
56 }
57
58 return ret;
59}
60
61EXPORT_SYMBOL(dma_alloc_coherent);
62
63void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
64 dma_addr_t dma_handle)
65{
66 free_pages((unsigned long) vaddr, get_order(size));
67}
68
69EXPORT_SYMBOL(dma_free_noncoherent);
70
71void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
72 dma_addr_t dma_handle)
73{
74 unsigned long addr = (unsigned long) vaddr;
75
76 addr = CAC_ADDR(addr);
77 free_pages(addr, get_order(size));
78}
79
80EXPORT_SYMBOL(dma_free_coherent);
81
82static inline void __dma_sync(unsigned long addr, size_t size,
83 enum dma_data_direction direction)
84{
85 switch (direction) {
86 case DMA_TO_DEVICE:
87 dma_cache_wback(addr, size);
88 break;
89
90 case DMA_FROM_DEVICE:
91 dma_cache_inv(addr, size);
92 break;
93
94 case DMA_BIDIRECTIONAL:
95 dma_cache_wback_inv(addr, size);
96 break;
97
98 default:
99 BUG();
100 }
101}
102
103dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
104 enum dma_data_direction direction)
105{
106 unsigned long addr = (unsigned long) ptr;
107
108 switch (direction) {
109 case DMA_TO_DEVICE:
110 dma_cache_wback(addr, size);
111 break;
112
113 case DMA_FROM_DEVICE:
114 dma_cache_inv(addr, size);
115 break;
116
117 case DMA_BIDIRECTIONAL:
118 dma_cache_wback_inv(addr, size);
119 break;
120
121 default:
122 BUG();
123 }
124
125 return virt_to_phys(ptr);
126}
127
128EXPORT_SYMBOL(dma_map_single);
129
130void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
131 enum dma_data_direction direction)
132{
133 unsigned long addr;
134 addr = dma_addr + PAGE_OFFSET;
135
136 switch (direction) {
137 case DMA_TO_DEVICE:
138 //dma_cache_wback(addr, size);
139 break;
140
141 case DMA_FROM_DEVICE:
142 //dma_cache_inv(addr, size);
143 break;
144
145 case DMA_BIDIRECTIONAL:
146 //dma_cache_wback_inv(addr, size);
147 break;
148
149 default:
150 BUG();
151 }
152}
153
154EXPORT_SYMBOL(dma_unmap_single);
155
156int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
157 enum dma_data_direction direction)
158{
159 int i;
160
161 BUG_ON(direction == DMA_NONE);
162
163 for (i = 0; i < nents; i++, sg++) {
164 unsigned long addr;
165
166 addr = (unsigned long) page_address(sg->page);
167 if (addr)
168 __dma_sync(addr + sg->offset, sg->length, direction);
169 sg->dma_address = (dma_addr_t)
170 (page_to_phys(sg->page) + sg->offset);
171 }
172
173 return nents;
174}
175
176EXPORT_SYMBOL(dma_map_sg);
177
178dma_addr_t dma_map_page(struct device *dev, struct page *page,
179 unsigned long offset, size_t size, enum dma_data_direction direction)
180{
181 unsigned long addr;
182
183 BUG_ON(direction == DMA_NONE);
184
185 addr = (unsigned long) page_address(page) + offset;
186 dma_cache_wback_inv(addr, size);
187
188 return page_to_phys(page) + offset;
189}
190
191EXPORT_SYMBOL(dma_map_page);
192
193void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
194 enum dma_data_direction direction)
195{
196 BUG_ON(direction == DMA_NONE);
197
198 if (direction != DMA_TO_DEVICE) {
199 unsigned long addr;
200
201 addr = dma_address + PAGE_OFFSET;
202 dma_cache_wback_inv(addr, size);
203 }
204}
205
206EXPORT_SYMBOL(dma_unmap_page);
207
208void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
209 enum dma_data_direction direction)
210{
211 unsigned long addr;
212 int i;
213
214 BUG_ON(direction == DMA_NONE);
215
216 if (direction == DMA_TO_DEVICE)
217 return;
218
219 for (i = 0; i < nhwentries; i++, sg++) {
220 addr = (unsigned long) page_address(sg->page);
221 if (!addr)
222 continue;
223 dma_cache_wback_inv(addr + sg->offset, sg->length);
224 }
225}
226
227EXPORT_SYMBOL(dma_unmap_sg);
228
229void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
230 size_t size, enum dma_data_direction direction)
231{
232 unsigned long addr;
233
234 BUG_ON(direction == DMA_NONE);
235
236 addr = dma_handle + PAGE_OFFSET;
237 __dma_sync(addr, size, direction);
238}
239
240EXPORT_SYMBOL(dma_sync_single_for_cpu);
241
242void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
243 size_t size, enum dma_data_direction direction)
244{
245 unsigned long addr;
246
247 BUG_ON(direction == DMA_NONE);
248
249 addr = dma_handle + PAGE_OFFSET;
250 __dma_sync(addr, size, direction);
251}
252
253EXPORT_SYMBOL(dma_sync_single_for_device);
254
255void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
256 unsigned long offset, size_t size, enum dma_data_direction direction)
257{
258 unsigned long addr;
259
260 BUG_ON(direction == DMA_NONE);
261
262 addr = dma_handle + offset + PAGE_OFFSET;
263 __dma_sync(addr, size, direction);
264}
265
266EXPORT_SYMBOL(dma_sync_single_range_for_cpu);
267
268void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle,
269 unsigned long offset, size_t size, enum dma_data_direction direction)
270{
271 unsigned long addr;
272
273 BUG_ON(direction == DMA_NONE);
274
275 addr = dma_handle + offset + PAGE_OFFSET;
276 __dma_sync(addr, size, direction);
277}
278
279EXPORT_SYMBOL(dma_sync_single_range_for_device);
280
281void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
282 enum dma_data_direction direction)
283{
284 int i;
285
286 BUG_ON(direction == DMA_NONE);
287
288 /* Make sure that gcc doesn't leave the empty loop body. */
289 for (i = 0; i < nelems; i++, sg++)
290 __dma_sync((unsigned long)page_address(sg->page),
291 sg->length, direction);
292}
293
294EXPORT_SYMBOL(dma_sync_sg_for_cpu);
295
296void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
297 enum dma_data_direction direction)
298{
299 int i;
300
301 BUG_ON(direction == DMA_NONE);
302
303 /* Make sure that gcc doesn't leave the empty loop body. */
304 for (i = 0; i < nelems; i++, sg++)
305 __dma_sync((unsigned long)page_address(sg->page),
306 sg->length, direction);
307}
308
309EXPORT_SYMBOL(dma_sync_sg_for_device);
310
311int dma_mapping_error(dma_addr_t dma_addr)
312{
313 return 0;
314}
315
316EXPORT_SYMBOL(dma_mapping_error);
317
318int dma_supported(struct device *dev, u64 mask)
319{
320 /*
321 * we fall back to GFP_DMA when the mask isn't all 1s,
322 * so we can't guarantee allocations that must be
323 * within a tighter range than GFP_DMA..
324 */
325 if (mask < 0x00ffffff)
326 return 0;
327
328 return 1;
329}
330
331EXPORT_SYMBOL(dma_supported);
332
333int dma_is_consistent(dma_addr_t dma_addr)
334{
335 return 1;
336}
337
338EXPORT_SYMBOL(dma_is_consistent);
339
340void dma_cache_sync(void *vaddr, size_t size, enum dma_data_direction direction)
341{
342 if (direction == DMA_NONE)
343 return;
344
345 dma_cache_wback_inv((unsigned long)vaddr, size);
346}
347
348EXPORT_SYMBOL(dma_cache_sync);
349
350/* The DAC routines are a PCIism.. */
351
352#ifdef CONFIG_PCI
353
354#include <linux/pci.h>
355
356dma64_addr_t pci_dac_page_to_dma(struct pci_dev *pdev,
357 struct page *page, unsigned long offset, int direction)
358{
359 return (dma64_addr_t)page_to_phys(page) + offset;
360}
361
362EXPORT_SYMBOL(pci_dac_page_to_dma);
363
364struct page *pci_dac_dma_to_page(struct pci_dev *pdev,
365 dma64_addr_t dma_addr)
366{
367 return mem_map + (dma_addr >> PAGE_SHIFT);
368}
369
370EXPORT_SYMBOL(pci_dac_dma_to_page);
371
372unsigned long pci_dac_dma_to_offset(struct pci_dev *pdev,
373 dma64_addr_t dma_addr)
374{
375 return dma_addr & ~PAGE_MASK;
376}
377
378EXPORT_SYMBOL(pci_dac_dma_to_offset);
379
380void pci_dac_dma_sync_single_for_cpu(struct pci_dev *pdev,
381 dma64_addr_t dma_addr, size_t len, int direction)
382{
383 BUG_ON(direction == PCI_DMA_NONE);
384
385 dma_cache_wback_inv(dma_addr + PAGE_OFFSET, len);
386}
387
388EXPORT_SYMBOL(pci_dac_dma_sync_single_for_cpu);
389
390void pci_dac_dma_sync_single_for_device(struct pci_dev *pdev,
391 dma64_addr_t dma_addr, size_t len, int direction)
392{
393 BUG_ON(direction == PCI_DMA_NONE);
394
395 dma_cache_wback_inv(dma_addr + PAGE_OFFSET, len);
396}
397
398EXPORT_SYMBOL(pci_dac_dma_sync_single_for_device);
399
400#endif /* CONFIG_PCI */
diff --git a/arch/mips/mm/extable.c b/arch/mips/mm/extable.c
new file mode 100644
index 000000000000..297fb9f390dc
--- /dev/null
+++ b/arch/mips/mm/extable.c
@@ -0,0 +1,21 @@
1/*
2 * linux/arch/mips/mm/extable.c
3 */
4#include <linux/module.h>
5#include <linux/spinlock.h>
6#include <asm/branch.h>
7#include <asm/uaccess.h>
8
9int fixup_exception(struct pt_regs *regs)
10{
11 const struct exception_table_entry *fixup;
12
13 fixup = search_exception_tables(exception_epc(regs));
14 if (fixup) {
15 regs->cp0_epc = fixup->nextinsn;
16
17 return 1;
18 }
19
20 return 0;
21}
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
new file mode 100644
index 000000000000..ec8077c74e9c
--- /dev/null
+++ b/arch/mips/mm/fault.c
@@ -0,0 +1,236 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1995 - 2000 by Ralf Baechle
7 */
8#include <linux/signal.h>
9#include <linux/sched.h>
10#include <linux/interrupt.h>
11#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/string.h>
14#include <linux/types.h>
15#include <linux/ptrace.h>
16#include <linux/mman.h>
17#include <linux/mm.h>
18#include <linux/smp.h>
19#include <linux/smp_lock.h>
20#include <linux/vt_kern.h> /* For unblank_screen() */
21#include <linux/module.h>
22
23#include <asm/branch.h>
24#include <asm/mmu_context.h>
25#include <asm/system.h>
26#include <asm/uaccess.h>
27#include <asm/ptrace.h>
28
29/*
30 * This routine handles page faults. It determines the address,
31 * and the problem, and then passes it off to one of the appropriate
32 * routines.
33 */
34asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
35 unsigned long address)
36{
37 struct vm_area_struct * vma = NULL;
38 struct task_struct *tsk = current;
39 struct mm_struct *mm = tsk->mm;
40 const int field = sizeof(unsigned long) * 2;
41 siginfo_t info;
42
43#if 0
44 printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", smp_processor_id(),
45 current->comm, current->pid, field, address, write,
46 field, regs->cp0_epc);
47#endif
48
49 info.si_code = SEGV_MAPERR;
50
51 /*
52 * We fault-in kernel-space virtual memory on-demand. The
53 * 'reference' page table is init_mm.pgd.
54 *
55 * NOTE! We MUST NOT take any locks for this case. We may
56 * be in an interrupt or a critical region, and should
57 * only copy the information from the master page table,
58 * nothing more.
59 */
60 if (unlikely(address >= VMALLOC_START))
61 goto vmalloc_fault;
62
63 /*
64 * If we're in an interrupt or have no user
65 * context, we must not take the fault..
66 */
67 if (in_atomic() || !mm)
68 goto bad_area_nosemaphore;
69
70 down_read(&mm->mmap_sem);
71 vma = find_vma(mm, address);
72 if (!vma)
73 goto bad_area;
74 if (vma->vm_start <= address)
75 goto good_area;
76 if (!(vma->vm_flags & VM_GROWSDOWN))
77 goto bad_area;
78 if (expand_stack(vma, address))
79 goto bad_area;
80/*
81 * Ok, we have a good vm_area for this memory access, so
82 * we can handle it..
83 */
84good_area:
85 info.si_code = SEGV_ACCERR;
86
87 if (write) {
88 if (!(vma->vm_flags & VM_WRITE))
89 goto bad_area;
90 } else {
91 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
92 goto bad_area;
93 }
94
95survive:
96 /*
97 * If for any reason at all we couldn't handle the fault,
98 * make sure we exit gracefully rather than endlessly redo
99 * the fault.
100 */
101 switch (handle_mm_fault(mm, vma, address, write)) {
102 case VM_FAULT_MINOR:
103 tsk->min_flt++;
104 break;
105 case VM_FAULT_MAJOR:
106 tsk->maj_flt++;
107 break;
108 case VM_FAULT_SIGBUS:
109 goto do_sigbus;
110 case VM_FAULT_OOM:
111 goto out_of_memory;
112 default:
113 BUG();
114 }
115
116 up_read(&mm->mmap_sem);
117 return;
118
119/*
120 * Something tried to access memory that isn't in our memory map..
121 * Fix it, but check if it's kernel or user first..
122 */
123bad_area:
124 up_read(&mm->mmap_sem);
125
126bad_area_nosemaphore:
127 /* User mode accesses just cause a SIGSEGV */
128 if (user_mode(regs)) {
129 tsk->thread.cp0_badvaddr = address;
130 tsk->thread.error_code = write;
131#if 0
132 printk("do_page_fault() #2: sending SIGSEGV to %s for "
133 "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
134 tsk->comm,
135 write ? "write access to" : "read access from",
136 field, address,
137 field, (unsigned long) regs->cp0_epc,
138 field, (unsigned long) regs->regs[31]);
139#endif
140 info.si_signo = SIGSEGV;
141 info.si_errno = 0;
142 /* info.si_code has been set above */
143 info.si_addr = (void *) address;
144 force_sig_info(SIGSEGV, &info, tsk);
145 return;
146 }
147
148no_context:
149 /* Are we prepared to handle this kernel fault? */
150 if (fixup_exception(regs)) {
151 current->thread.cp0_baduaddr = address;
152 return;
153 }
154
155 /*
156 * Oops. The kernel tried to access some bad page. We'll have to
157 * terminate things with extreme prejudice.
158 */
159
160 bust_spinlocks(1);
161
162 printk(KERN_ALERT "CPU %d Unable to handle kernel paging request at "
163 "virtual address %0*lx, epc == %0*lx, ra == %0*lx\n",
164 smp_processor_id(), field, address, field, regs->cp0_epc,
165 field, regs->regs[31]);
166 die("Oops", regs);
167
168/*
169 * We ran out of memory, or some other thing happened to us that made
170 * us unable to handle the page fault gracefully.
171 */
172out_of_memory:
173 up_read(&mm->mmap_sem);
174 if (tsk->pid == 1) {
175 yield();
176 down_read(&mm->mmap_sem);
177 goto survive;
178 }
179 printk("VM: killing process %s\n", tsk->comm);
180 if (user_mode(regs))
181 do_exit(SIGKILL);
182 goto no_context;
183
184do_sigbus:
185 up_read(&mm->mmap_sem);
186
187 /* Kernel mode? Handle exceptions or die */
188 if (!user_mode(regs))
189 goto no_context;
190
191 /*
192 * Send a sigbus, regardless of whether we were in kernel
193 * or user mode.
194 */
195 tsk->thread.cp0_badvaddr = address;
196 info.si_signo = SIGBUS;
197 info.si_errno = 0;
198 info.si_code = BUS_ADRERR;
199 info.si_addr = (void *) address;
200 force_sig_info(SIGBUS, &info, tsk);
201
202 return;
203
204vmalloc_fault:
205 {
206 /*
207 * Synchronize this task's top level page-table
208 * with the 'reference' page table.
209 *
210 * Do _not_ use "tsk" here. We might be inside
211 * an interrupt in the middle of a task switch..
212 */
213 int offset = __pgd_offset(address);
214 pgd_t *pgd, *pgd_k;
215 pmd_t *pmd, *pmd_k;
216 pte_t *pte_k;
217
218 pgd = (pgd_t *) pgd_current[smp_processor_id()] + offset;
219 pgd_k = init_mm.pgd + offset;
220
221 if (!pgd_present(*pgd_k))
222 goto no_context;
223 set_pgd(pgd, *pgd_k);
224
225 pmd = pmd_offset(pgd, address);
226 pmd_k = pmd_offset(pgd_k, address);
227 if (!pmd_present(*pmd_k))
228 goto no_context;
229 set_pmd(pmd, *pmd_k);
230
231 pte_k = pte_offset_kernel(pmd_k, address);
232 if (!pte_present(*pte_k))
233 goto no_context;
234 return;
235 }
236}
diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
new file mode 100644
index 000000000000..dd5e2e31885b
--- /dev/null
+++ b/arch/mips/mm/highmem.c
@@ -0,0 +1,103 @@
1#include <linux/config.h>
2#include <linux/module.h>
3#include <linux/highmem.h>
4#include <asm/tlbflush.h>
5
6void *__kmap(struct page *page)
7{
8 void *addr;
9
10 might_sleep();
11 if (!PageHighMem(page))
12 return page_address(page);
13 addr = kmap_high(page);
14 flush_tlb_one((unsigned long)addr);
15
16 return addr;
17}
18
19void __kunmap(struct page *page)
20{
21 if (in_interrupt())
22 BUG();
23 if (!PageHighMem(page))
24 return;
25 kunmap_high(page);
26}
27
28/*
29 * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because
30 * no global lock is needed and because the kmap code must perform a global TLB
31 * invalidation when the kmap pool wraps.
32 *
33 * However when holding an atomic kmap is is not legal to sleep, so atomic
34 * kmaps are appropriate for short, tight code paths only.
35 */
36
37void *__kmap_atomic(struct page *page, enum km_type type)
38{
39 enum fixed_addresses idx;
40 unsigned long vaddr;
41
42 /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
43 inc_preempt_count();
44 if (!PageHighMem(page))
45 return page_address(page);
46
47 idx = type + KM_TYPE_NR*smp_processor_id();
48 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
49#ifdef CONFIG_DEBUG_HIGHMEM
50 if (!pte_none(*(kmap_pte-idx)))
51 BUG();
52#endif
53 set_pte(kmap_pte-idx, mk_pte(page, kmap_prot));
54 local_flush_tlb_one((unsigned long)vaddr);
55
56 return (void*) vaddr;
57}
58
59void __kunmap_atomic(void *kvaddr, enum km_type type)
60{
61#ifdef CONFIG_DEBUG_HIGHMEM
62 unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
63 enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id();
64
65 if (vaddr < FIXADDR_START) { // FIXME
66 dec_preempt_count();
67 preempt_check_resched();
68 return;
69 }
70
71 if (vaddr != __fix_to_virt(FIX_KMAP_BEGIN+idx))
72 BUG();
73
74 /*
75 * force other mappings to Oops if they'll try to access
76 * this pte without first remap it
77 */
78 pte_clear(&init_mm, vaddr, kmap_pte-idx);
79 local_flush_tlb_one(vaddr);
80#endif
81
82 dec_preempt_count();
83 preempt_check_resched();
84}
85
86struct page *__kmap_atomic_to_page(void *ptr)
87{
88 unsigned long idx, vaddr = (unsigned long)ptr;
89 pte_t *pte;
90
91 if (vaddr < FIXADDR_START)
92 return virt_to_page(ptr);
93
94 idx = virt_to_fix(vaddr);
95 pte = kmap_pte - (idx - FIX_KMAP_BEGIN);
96 return pte_page(*pte);
97}
98
99EXPORT_SYMBOL(__kmap);
100EXPORT_SYMBOL(__kunmap);
101EXPORT_SYMBOL(__kmap_atomic);
102EXPORT_SYMBOL(__kunmap_atomic);
103EXPORT_SYMBOL(__kmap_atomic_to_page);
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
new file mode 100644
index 000000000000..b027ce7efbc6
--- /dev/null
+++ b/arch/mips/mm/init.c
@@ -0,0 +1,304 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1994 - 2000 Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
9 * Copyright (C) 2000 MIPS Technologies, Inc. All rights reserved.
10 */
11#include <linux/config.h>
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/signal.h>
15#include <linux/sched.h>
16#include <linux/kernel.h>
17#include <linux/errno.h>
18#include <linux/string.h>
19#include <linux/types.h>
20#include <linux/pagemap.h>
21#include <linux/ptrace.h>
22#include <linux/mman.h>
23#include <linux/mm.h>
24#include <linux/bootmem.h>
25#include <linux/highmem.h>
26#include <linux/swap.h>
27
28#include <asm/bootinfo.h>
29#include <asm/cachectl.h>
30#include <asm/cpu.h>
31#include <asm/dma.h>
32#include <asm/mmu_context.h>
33#include <asm/sections.h>
34#include <asm/pgtable.h>
35#include <asm/pgalloc.h>
36#include <asm/tlb.h>
37
38DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
39
40unsigned long highstart_pfn, highend_pfn;
41
42/*
43 * We have up to 8 empty zeroed pages so we can map one of the right colour
44 * when needed. This is necessary only on R4000 / R4400 SC and MC versions
45 * where we have to avoid VCED / VECI exceptions for good performance at
46 * any price. Since page is never written to after the initialization we
47 * don't have to care about aliases on other CPUs.
48 */
49unsigned long empty_zero_page, zero_page_mask;
50
51/*
52 * Not static inline because used by IP27 special magic initialization code
53 */
54unsigned long setup_zero_pages(void)
55{
56 unsigned long order, size;
57 struct page *page;
58
59 if (cpu_has_vce)
60 order = 3;
61 else
62 order = 0;
63
64 empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
65 if (!empty_zero_page)
66 panic("Oh boy, that early out of memory?");
67
68 page = virt_to_page(empty_zero_page);
69 while (page < virt_to_page(empty_zero_page + (PAGE_SIZE << order))) {
70 set_bit(PG_reserved, &page->flags);
71 set_page_count(page, 0);
72 page++;
73 }
74
75 size = PAGE_SIZE << order;
76 zero_page_mask = (size - 1) & PAGE_MASK;
77
78 return 1UL << order;
79}
80
81#ifdef CONFIG_HIGHMEM
82pte_t *kmap_pte;
83pgprot_t kmap_prot;
84
85#define kmap_get_fixmap_pte(vaddr) \
86 pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
87
88static void __init kmap_init(void)
89{
90 unsigned long kmap_vstart;
91
92 /* cache the first kmap pte */
93 kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
94 kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
95
96 kmap_prot = PAGE_KERNEL;
97}
98
99#ifdef CONFIG_MIPS64
100static void __init fixrange_init(unsigned long start, unsigned long end,
101 pgd_t *pgd_base)
102{
103 pgd_t *pgd;
104 pmd_t *pmd;
105 pte_t *pte;
106 int i, j;
107 unsigned long vaddr;
108
109 vaddr = start;
110 i = __pgd_offset(vaddr);
111 j = __pmd_offset(vaddr);
112 pgd = pgd_base + i;
113
114 for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
115 pmd = (pmd_t *)pgd;
116 for (; (j < PTRS_PER_PMD) && (vaddr != end); pmd++, j++) {
117 if (pmd_none(*pmd)) {
118 pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
119 set_pmd(pmd, __pmd(pte));
120 if (pte != pte_offset_kernel(pmd, 0))
121 BUG();
122 }
123 vaddr += PMD_SIZE;
124 }
125 j = 0;
126 }
127}
128#endif /* CONFIG_MIPS64 */
129#endif /* CONFIG_HIGHMEM */
130
131#ifndef CONFIG_DISCONTIGMEM
132extern void pagetable_init(void);
133
134void __init paging_init(void)
135{
136 unsigned long zones_size[MAX_NR_ZONES] = {0, 0, 0};
137 unsigned long max_dma, high, low;
138
139 pagetable_init();
140
141#ifdef CONFIG_HIGHMEM
142 kmap_init();
143#endif
144
145 max_dma = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
146 low = max_low_pfn;
147 high = highend_pfn;
148
149#ifdef CONFIG_ISA
150 if (low < max_dma)
151 zones_size[ZONE_DMA] = low;
152 else {
153 zones_size[ZONE_DMA] = max_dma;
154 zones_size[ZONE_NORMAL] = low - max_dma;
155 }
156#else
157 zones_size[ZONE_DMA] = low;
158#endif
159#ifdef CONFIG_HIGHMEM
160 if (cpu_has_dc_aliases) {
161 printk(KERN_WARNING "This processor doesn't support highmem.");
162 if (high - low)
163 printk(" %ldk highmem ignored", high - low);
164 printk("\n");
165 } else
166 zones_size[ZONE_HIGHMEM] = high - low;
167#endif
168
169 free_area_init(zones_size);
170}
171
172#define PFN_UP(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT)
173#define PFN_DOWN(x) ((x) >> PAGE_SHIFT)
174
175static inline int page_is_ram(unsigned long pagenr)
176{
177 int i;
178
179 for (i = 0; i < boot_mem_map.nr_map; i++) {
180 unsigned long addr, end;
181
182 if (boot_mem_map.map[i].type != BOOT_MEM_RAM)
183 /* not usable memory */
184 continue;
185
186 addr = PFN_UP(boot_mem_map.map[i].addr);
187 end = PFN_DOWN(boot_mem_map.map[i].addr +
188 boot_mem_map.map[i].size);
189
190 if (pagenr >= addr && pagenr < end)
191 return 1;
192 }
193
194 return 0;
195}
196
197void __init mem_init(void)
198{
199 unsigned long codesize, reservedpages, datasize, initsize;
200 unsigned long tmp, ram;
201
202#ifdef CONFIG_HIGHMEM
203#ifdef CONFIG_DISCONTIGMEM
204#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
205#endif
206 max_mapnr = num_physpages = highend_pfn;
207#else
208 max_mapnr = num_physpages = max_low_pfn;
209#endif
210 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
211
212 totalram_pages += free_all_bootmem();
213 totalram_pages -= setup_zero_pages(); /* Setup zeroed pages. */
214
215 reservedpages = ram = 0;
216 for (tmp = 0; tmp < max_low_pfn; tmp++)
217 if (page_is_ram(tmp)) {
218 ram++;
219 if (PageReserved(mem_map+tmp))
220 reservedpages++;
221 }
222
223#ifdef CONFIG_HIGHMEM
224 for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
225 struct page *page = mem_map + tmp;
226
227 if (!page_is_ram(tmp)) {
228 SetPageReserved(page);
229 continue;
230 }
231 ClearPageReserved(page);
232#ifdef CONFIG_LIMITED_DMA
233 set_page_address(page, lowmem_page_address(page));
234#endif
235 set_bit(PG_highmem, &page->flags);
236 set_page_count(page, 1);
237 __free_page(page);
238 totalhigh_pages++;
239 }
240 totalram_pages += totalhigh_pages;
241#endif
242
243 codesize = (unsigned long) &_etext - (unsigned long) &_text;
244 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
245 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
246
247 printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
248 "%ldk reserved, %ldk data, %ldk init, %ldk highmem)\n",
249 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
250 ram << (PAGE_SHIFT-10),
251 codesize >> 10,
252 reservedpages << (PAGE_SHIFT-10),
253 datasize >> 10,
254 initsize >> 10,
255 (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)));
256}
257#endif /* !CONFIG_DISCONTIGMEM */
258
259#ifdef CONFIG_BLK_DEV_INITRD
260void free_initrd_mem(unsigned long start, unsigned long end)
261{
262#ifdef CONFIG_MIPS64
263 /* Switch from KSEG0 to XKPHYS addresses */
264 start = (unsigned long)phys_to_virt(CPHYSADDR(start));
265 end = (unsigned long)phys_to_virt(CPHYSADDR(end));
266#endif
267 if (start < end)
268 printk(KERN_INFO "Freeing initrd memory: %ldk freed\n",
269 (end - start) >> 10);
270
271 for (; start < end; start += PAGE_SIZE) {
272 ClearPageReserved(virt_to_page(start));
273 set_page_count(virt_to_page(start), 1);
274 free_page(start);
275 totalram_pages++;
276 }
277}
278#endif
279
280extern unsigned long prom_free_prom_memory(void);
281
282void free_initmem(void)
283{
284 unsigned long addr, page, freed;
285
286 freed = prom_free_prom_memory();
287
288 addr = (unsigned long) &__init_begin;
289 while (addr < (unsigned long) &__init_end) {
290#ifdef CONFIG_MIPS64
291 page = PAGE_OFFSET | CPHYSADDR(addr);
292#else
293 page = addr;
294#endif
295 ClearPageReserved(virt_to_page(page));
296 set_page_count(virt_to_page(page), 1);
297 free_page(page);
298 totalram_pages++;
299 freed += PAGE_SIZE;
300 addr += PAGE_SIZE;
301 }
302 printk(KERN_INFO "Freeing unused kernel memory: %ldk freed\n",
303 freed >> 10);
304}
diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c
new file mode 100644
index 000000000000..adf352273f63
--- /dev/null
+++ b/arch/mips/mm/ioremap.c
@@ -0,0 +1,202 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * (C) Copyright 1995 1996 Linus Torvalds
7 * (C) Copyright 2001, 2002 Ralf Baechle
8 */
9#include <linux/module.h>
10#include <asm/addrspace.h>
11#include <asm/byteorder.h>
12
13#include <linux/vmalloc.h>
14#include <asm/cacheflush.h>
15#include <asm/io.h>
16#include <asm/tlbflush.h>
17
18static inline void remap_area_pte(pte_t * pte, unsigned long address,
19 phys_t size, phys_t phys_addr, unsigned long flags)
20{
21 phys_t end;
22 unsigned long pfn;
23 pgprot_t pgprot = __pgprot(_PAGE_GLOBAL | _PAGE_PRESENT | __READABLE
24 | __WRITEABLE | flags);
25
26 address &= ~PMD_MASK;
27 end = address + size;
28 if (end > PMD_SIZE)
29 end = PMD_SIZE;
30 if (address >= end)
31 BUG();
32 pfn = phys_addr >> PAGE_SHIFT;
33 do {
34 if (!pte_none(*pte)) {
35 printk("remap_area_pte: page already exists\n");
36 BUG();
37 }
38 set_pte(pte, pfn_pte(pfn, pgprot));
39 address += PAGE_SIZE;
40 pfn++;
41 pte++;
42 } while (address && (address < end));
43}
44
45static inline int remap_area_pmd(pmd_t * pmd, unsigned long address,
46 phys_t size, phys_t phys_addr, unsigned long flags)
47{
48 phys_t end;
49
50 address &= ~PGDIR_MASK;
51 end = address + size;
52 if (end > PGDIR_SIZE)
53 end = PGDIR_SIZE;
54 phys_addr -= address;
55 if (address >= end)
56 BUG();
57 do {
58 pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
59 if (!pte)
60 return -ENOMEM;
61 remap_area_pte(pte, address, end - address, address + phys_addr, flags);
62 address = (address + PMD_SIZE) & PMD_MASK;
63 pmd++;
64 } while (address && (address < end));
65 return 0;
66}
67
68static int remap_area_pages(unsigned long address, phys_t phys_addr,
69 phys_t size, unsigned long flags)
70{
71 int error;
72 pgd_t * dir;
73 unsigned long end = address + size;
74
75 phys_addr -= address;
76 dir = pgd_offset(&init_mm, address);
77 flush_cache_all();
78 if (address >= end)
79 BUG();
80 spin_lock(&init_mm.page_table_lock);
81 do {
82 pmd_t *pmd;
83 pmd = pmd_alloc(&init_mm, dir, address);
84 error = -ENOMEM;
85 if (!pmd)
86 break;
87 if (remap_area_pmd(pmd, address, end - address,
88 phys_addr + address, flags))
89 break;
90 error = 0;
91 address = (address + PGDIR_SIZE) & PGDIR_MASK;
92 dir++;
93 } while (address && (address < end));
94 spin_unlock(&init_mm.page_table_lock);
95 flush_tlb_all();
96 return error;
97}
98
99/*
100 * Allow physical addresses to be fixed up to help 36 bit peripherals.
101 */
102phys_t __attribute__ ((weak))
103fixup_bigphys_addr(phys_t phys_addr, phys_t size)
104{
105 return phys_addr;
106}
107
108/*
109 * Generic mapping function (not visible outside):
110 */
111
112/*
113 * Remap an arbitrary physical address space into the kernel virtual
114 * address space. Needed when the kernel wants to access high addresses
115 * directly.
116 *
117 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
118 * have to convert them into an offset in a page-aligned mapping, but the
119 * caller shouldn't need to know that small detail.
120 */
121
122#define IS_LOW512(addr) (!((phys_t)(addr) & (phys_t) ~0x1fffffffULL))
123
124void * __ioremap(phys_t phys_addr, phys_t size, unsigned long flags)
125{
126 struct vm_struct * area;
127 unsigned long offset;
128 phys_t last_addr;
129 void * addr;
130
131 phys_addr = fixup_bigphys_addr(phys_addr, size);
132
133 /* Don't allow wraparound or zero size */
134 last_addr = phys_addr + size - 1;
135 if (!size || last_addr < phys_addr)
136 return NULL;
137
138 /*
139 * Map uncached objects in the low 512mb of address space using KSEG1,
140 * otherwise map using page tables.
141 */
142 if (IS_LOW512(phys_addr) && IS_LOW512(last_addr) &&
143 flags == _CACHE_UNCACHED)
144 return (void *) KSEG1ADDR(phys_addr);
145
146 /*
147 * Don't allow anybody to remap normal RAM that we're using..
148 */
149 if (phys_addr < virt_to_phys(high_memory)) {
150 char *t_addr, *t_end;
151 struct page *page;
152
153 t_addr = __va(phys_addr);
154 t_end = t_addr + (size - 1);
155
156 for(page = virt_to_page(t_addr); page <= virt_to_page(t_end); page++)
157 if(!PageReserved(page))
158 return NULL;
159 }
160
161 /*
162 * Mappings have to be page-aligned
163 */
164 offset = phys_addr & ~PAGE_MASK;
165 phys_addr &= PAGE_MASK;
166 size = PAGE_ALIGN(last_addr + 1) - phys_addr;
167
168 /*
169 * Ok, go for it..
170 */
171 area = get_vm_area(size, VM_IOREMAP);
172 if (!area)
173 return NULL;
174 addr = area->addr;
175 if (remap_area_pages((unsigned long) addr, phys_addr, size, flags)) {
176 vunmap(addr);
177 return NULL;
178 }
179
180 return (void *) (offset + (char *)addr);
181}
182
183#define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == KSEG1)
184
185void __iounmap(volatile void __iomem *addr)
186{
187 struct vm_struct *p;
188
189 if (IS_KSEG1(addr))
190 return;
191
192 p = remove_vm_area((void *) (PAGE_MASK & (unsigned long __force) addr));
193 if (!p) {
194 printk(KERN_ERR "iounmap: bad address %p\n", addr);
195 return;
196 }
197
198 kfree(p);
199}
200
201EXPORT_SYMBOL(__ioremap);
202EXPORT_SYMBOL(__iounmap);
diff --git a/arch/mips/mm/pg-r4k.c b/arch/mips/mm/pg-r4k.c
new file mode 100644
index 000000000000..9f8b16541577
--- /dev/null
+++ b/arch/mips/mm/pg-r4k.c
@@ -0,0 +1,489 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
7 */
8#include <linux/init.h>
9#include <linux/kernel.h>
10#include <linux/sched.h>
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/proc_fs.h>
14
15#include <asm/cacheops.h>
16#include <asm/inst.h>
17#include <asm/io.h>
18#include <asm/page.h>
19#include <asm/pgtable.h>
20#include <asm/prefetch.h>
21#include <asm/system.h>
22#include <asm/bootinfo.h>
23#include <asm/mipsregs.h>
24#include <asm/mmu_context.h>
25#include <asm/cpu.h>
26#include <asm/war.h>
27
28#define half_scache_line_size() (cpu_scache_line_size() >> 1)
29
30/*
31 * Maximum sizes:
32 *
33 * R4000 128 bytes S-cache: 0x58 bytes
34 * R4600 v1.7: 0x5c bytes
35 * R4600 v2.0: 0x60 bytes
36 * With prefetching, 16 byte strides 0xa0 bytes
37 */
38
39static unsigned int clear_page_array[0x130 / 4];
40
41void clear_page(void * page) __attribute__((alias("clear_page_array")));
42
43EXPORT_SYMBOL(clear_page);
44
45/*
46 * Maximum sizes:
47 *
48 * R4000 128 bytes S-cache: 0x11c bytes
49 * R4600 v1.7: 0x080 bytes
50 * R4600 v2.0: 0x07c bytes
51 * With prefetching, 16 byte strides 0x0b8 bytes
52 */
53static unsigned int copy_page_array[0x148 / 4];
54
55void copy_page(void *to, void *from) __attribute__((alias("copy_page_array")));
56
57EXPORT_SYMBOL(copy_page);
58
59/*
60 * This is suboptimal for 32-bit kernels; we assume that R10000 is only used
61 * with 64-bit kernels. The prefetch offsets have been experimentally tuned
62 * an Origin 200.
63 */
64static int pref_offset_clear __initdata = 512;
65static int pref_offset_copy __initdata = 256;
66
67static unsigned int pref_src_mode __initdata;
68static unsigned int pref_dst_mode __initdata;
69
70static int load_offset __initdata;
71static int store_offset __initdata;
72
73static unsigned int __initdata *dest, *epc;
74
75static unsigned int instruction_pending;
76static union mips_instruction delayed_mi;
77
78static void __init emit_instruction(union mips_instruction mi)
79{
80 if (instruction_pending)
81 *epc++ = delayed_mi.word;
82
83 instruction_pending = 1;
84 delayed_mi = mi;
85}
86
87static inline void flush_delay_slot_or_nop(void)
88{
89 if (instruction_pending) {
90 *epc++ = delayed_mi.word;
91 instruction_pending = 0;
92 return;
93 }
94
95 *epc++ = 0;
96}
97
98static inline unsigned int *label(void)
99{
100 if (instruction_pending) {
101 *epc++ = delayed_mi.word;
102 instruction_pending = 0;
103 }
104
105 return epc;
106}
107
108static inline void build_insn_word(unsigned int word)
109{
110 union mips_instruction mi;
111
112 mi.word = word;
113
114 emit_instruction(mi);
115}
116
117static inline void build_nop(void)
118{
119 build_insn_word(0); /* nop */
120}
121
122static inline void build_src_pref(int advance)
123{
124 if (!(load_offset & (cpu_dcache_line_size() - 1))) {
125 union mips_instruction mi;
126
127 mi.i_format.opcode = pref_op;
128 mi.i_format.rs = 5; /* $a1 */
129 mi.i_format.rt = pref_src_mode;
130 mi.i_format.simmediate = load_offset + advance;
131
132 emit_instruction(mi);
133 }
134}
135
136static inline void __build_load_reg(int reg)
137{
138 union mips_instruction mi;
139 unsigned int width;
140
141 if (cpu_has_64bit_gp_regs) {
142 mi.i_format.opcode = ld_op;
143 width = 8;
144 } else {
145 mi.i_format.opcode = lw_op;
146 width = 4;
147 }
148 mi.i_format.rs = 5; /* $a1 */
149 mi.i_format.rt = reg; /* $reg */
150 mi.i_format.simmediate = load_offset;
151
152 load_offset += width;
153 emit_instruction(mi);
154}
155
156static inline void build_load_reg(int reg)
157{
158 if (cpu_has_prefetch)
159 build_src_pref(pref_offset_copy);
160
161 __build_load_reg(reg);
162}
163
164static inline void build_dst_pref(int advance)
165{
166 if (!(store_offset & (cpu_dcache_line_size() - 1))) {
167 union mips_instruction mi;
168
169 mi.i_format.opcode = pref_op;
170 mi.i_format.rs = 4; /* $a0 */
171 mi.i_format.rt = pref_dst_mode;
172 mi.i_format.simmediate = store_offset + advance;
173
174 emit_instruction(mi);
175 }
176}
177
178static inline void build_cdex_s(void)
179{
180 union mips_instruction mi;
181
182 if ((store_offset & (cpu_scache_line_size() - 1)))
183 return;
184
185 mi.c_format.opcode = cache_op;
186 mi.c_format.rs = 4; /* $a0 */
187 mi.c_format.c_op = 3; /* Create Dirty Exclusive */
188 mi.c_format.cache = 3; /* Secondary Data Cache */
189 mi.c_format.simmediate = store_offset;
190
191 emit_instruction(mi);
192}
193
194static inline void build_cdex_p(void)
195{
196 union mips_instruction mi;
197
198 if (store_offset & (cpu_dcache_line_size() - 1))
199 return;
200
201 if (R4600_V1_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2010)) {
202 build_nop();
203 build_nop();
204 build_nop();
205 build_nop();
206 }
207
208 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
209 build_insn_word(0x8c200000); /* lw $zero, ($at) */
210
211 mi.c_format.opcode = cache_op;
212 mi.c_format.rs = 4; /* $a0 */
213 mi.c_format.c_op = 3; /* Create Dirty Exclusive */
214 mi.c_format.cache = 1; /* Data Cache */
215 mi.c_format.simmediate = store_offset;
216
217 emit_instruction(mi);
218}
219
220static void __init __build_store_reg(int reg)
221{
222 union mips_instruction mi;
223 unsigned int width;
224
225 if (cpu_has_64bit_gp_regs ||
226 (cpu_has_64bit_zero_reg && reg == 0)) {
227 mi.i_format.opcode = sd_op;
228 width = 8;
229 } else {
230 mi.i_format.opcode = sw_op;
231 width = 4;
232 }
233 mi.i_format.rs = 4; /* $a0 */
234 mi.i_format.rt = reg; /* $reg */
235 mi.i_format.simmediate = store_offset;
236
237 store_offset += width;
238 emit_instruction(mi);
239}
240
241static inline void build_store_reg(int reg)
242{
243 if (cpu_has_prefetch)
244 if (reg)
245 build_dst_pref(pref_offset_copy);
246 else
247 build_dst_pref(pref_offset_clear);
248 else if (cpu_has_cache_cdex_s)
249 build_cdex_s();
250 else if (cpu_has_cache_cdex_p)
251 build_cdex_p();
252
253 __build_store_reg(reg);
254}
255
256static inline void build_addiu_a2_a0(unsigned long offset)
257{
258 union mips_instruction mi;
259
260 BUG_ON(offset > 0x7fff);
261
262 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
263 mi.i_format.rs = 4; /* $a0 */
264 mi.i_format.rt = 6; /* $a2 */
265 mi.i_format.simmediate = offset;
266
267 emit_instruction(mi);
268}
269
270static inline void build_addiu_a1(unsigned long offset)
271{
272 union mips_instruction mi;
273
274 BUG_ON(offset > 0x7fff);
275
276 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
277 mi.i_format.rs = 5; /* $a1 */
278 mi.i_format.rt = 5; /* $a1 */
279 mi.i_format.simmediate = offset;
280
281 load_offset -= offset;
282
283 emit_instruction(mi);
284}
285
286static inline void build_addiu_a0(unsigned long offset)
287{
288 union mips_instruction mi;
289
290 BUG_ON(offset > 0x7fff);
291
292 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
293 mi.i_format.rs = 4; /* $a0 */
294 mi.i_format.rt = 4; /* $a0 */
295 mi.i_format.simmediate = offset;
296
297 store_offset -= offset;
298
299 emit_instruction(mi);
300}
301
302static inline void build_bne(unsigned int *dest)
303{
304 union mips_instruction mi;
305
306 mi.i_format.opcode = bne_op;
307 mi.i_format.rs = 6; /* $a2 */
308 mi.i_format.rt = 4; /* $a0 */
309 mi.i_format.simmediate = dest - epc - 1;
310
311 *epc++ = mi.word;
312 flush_delay_slot_or_nop();
313}
314
315static inline void build_jr_ra(void)
316{
317 union mips_instruction mi;
318
319 mi.r_format.opcode = spec_op;
320 mi.r_format.rs = 31;
321 mi.r_format.rt = 0;
322 mi.r_format.rd = 0;
323 mi.r_format.re = 0;
324 mi.r_format.func = jr_op;
325
326 *epc++ = mi.word;
327 flush_delay_slot_or_nop();
328}
329
330void __init build_clear_page(void)
331{
332 unsigned int loop_start;
333
334 epc = (unsigned int *) &clear_page_array;
335 instruction_pending = 0;
336 store_offset = 0;
337
338 if (cpu_has_prefetch) {
339 switch (current_cpu_data.cputype) {
340 case CPU_RM9000:
341 /*
342 * As a workaround for erratum G105 which make the
343 * PrepareForStore hint unusable we fall back to
344 * StoreRetained on the RM9000. Once it is known which
345 * versions of the RM9000 we'll be able to condition-
346 * alize this.
347 */
348
349 case CPU_R10000:
350 case CPU_R12000:
351 pref_src_mode = Pref_LoadStreamed;
352 pref_dst_mode = Pref_StoreStreamed;
353 break;
354
355 default:
356 pref_src_mode = Pref_LoadStreamed;
357 pref_dst_mode = Pref_PrepareForStore;
358 break;
359 }
360 }
361
362 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0));
363
364 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
365 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
366
367dest = label();
368 do {
369 build_store_reg(0);
370 build_store_reg(0);
371 build_store_reg(0);
372 build_store_reg(0);
373 } while (store_offset < half_scache_line_size());
374 build_addiu_a0(2 * store_offset);
375 loop_start = store_offset;
376 do {
377 build_store_reg(0);
378 build_store_reg(0);
379 build_store_reg(0);
380 build_store_reg(0);
381 } while ((store_offset - loop_start) < half_scache_line_size());
382 build_bne(dest);
383
384 if (cpu_has_prefetch && pref_offset_clear) {
385 build_addiu_a2_a0(pref_offset_clear);
386 dest = label();
387 loop_start = store_offset;
388 do {
389 __build_store_reg(0);
390 __build_store_reg(0);
391 __build_store_reg(0);
392 __build_store_reg(0);
393 } while ((store_offset - loop_start) < half_scache_line_size());
394 build_addiu_a0(2 * store_offset);
395 loop_start = store_offset;
396 do {
397 __build_store_reg(0);
398 __build_store_reg(0);
399 __build_store_reg(0);
400 __build_store_reg(0);
401 } while ((store_offset - loop_start) < half_scache_line_size());
402 build_bne(dest);
403 }
404
405 build_jr_ra();
406
407 flush_icache_range((unsigned long)&clear_page_array,
408 (unsigned long) epc);
409
410 BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array));
411}
412
413void __init build_copy_page(void)
414{
415 unsigned int loop_start;
416
417 epc = (unsigned int *) &copy_page_array;
418 store_offset = load_offset = 0;
419 instruction_pending = 0;
420
421 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0));
422
423 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
424 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
425
426dest = label();
427 loop_start = store_offset;
428 do {
429 build_load_reg( 8);
430 build_load_reg( 9);
431 build_load_reg(10);
432 build_load_reg(11);
433 build_store_reg( 8);
434 build_store_reg( 9);
435 build_store_reg(10);
436 build_store_reg(11);
437 } while ((store_offset - loop_start) < half_scache_line_size());
438 build_addiu_a0(2 * store_offset);
439 build_addiu_a1(2 * load_offset);
440 loop_start = store_offset;
441 do {
442 build_load_reg( 8);
443 build_load_reg( 9);
444 build_load_reg(10);
445 build_load_reg(11);
446 build_store_reg( 8);
447 build_store_reg( 9);
448 build_store_reg(10);
449 build_store_reg(11);
450 } while ((store_offset - loop_start) < half_scache_line_size());
451 build_bne(dest);
452
453 if (cpu_has_prefetch && pref_offset_copy) {
454 build_addiu_a2_a0(pref_offset_copy);
455 dest = label();
456 loop_start = store_offset;
457 do {
458 __build_load_reg( 8);
459 __build_load_reg( 9);
460 __build_load_reg(10);
461 __build_load_reg(11);
462 __build_store_reg( 8);
463 __build_store_reg( 9);
464 __build_store_reg(10);
465 __build_store_reg(11);
466 } while ((store_offset - loop_start) < half_scache_line_size());
467 build_addiu_a0(2 * store_offset);
468 build_addiu_a1(2 * load_offset);
469 loop_start = store_offset;
470 do {
471 __build_load_reg( 8);
472 __build_load_reg( 9);
473 __build_load_reg(10);
474 __build_load_reg(11);
475 __build_store_reg( 8);
476 __build_store_reg( 9);
477 __build_store_reg(10);
478 __build_store_reg(11);
479 } while ((store_offset - loop_start) < half_scache_line_size());
480 build_bne(dest);
481 }
482
483 build_jr_ra();
484
485 flush_icache_range((unsigned long)&copy_page_array,
486 (unsigned long) epc);
487
488 BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array));
489}
diff --git a/arch/mips/mm/pg-sb1.c b/arch/mips/mm/pg-sb1.c
new file mode 100644
index 000000000000..59d131b5e536
--- /dev/null
+++ b/arch/mips/mm/pg-sb1.c
@@ -0,0 +1,287 @@
1/*
2 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
3 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org)
4 * Copyright (C) 2000 SiByte, Inc.
5 * Copyright (C) 2005 Thiemo Seufer
6 *
7 * Written by Justin Carlson of SiByte, Inc.
8 * and Kip Walker of Broadcom Corp.
9 *
10 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version 2
14 * of the License, or (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
24 */
25#include <linux/config.h>
26#include <linux/module.h>
27#include <linux/sched.h>
28#include <linux/smp.h>
29
30#include <asm/io.h>
31#include <asm/sibyte/sb1250.h>
32#include <asm/sibyte/sb1250_regs.h>
33#include <asm/sibyte/sb1250_dma.h>
34
35#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
36#define SB1_PREF_LOAD_STREAMED_HINT "0"
37#define SB1_PREF_STORE_STREAMED_HINT "1"
38#else
39#define SB1_PREF_LOAD_STREAMED_HINT "4"
40#define SB1_PREF_STORE_STREAMED_HINT "5"
41#endif
42
43static inline void clear_page_cpu(void *page)
44{
45 unsigned char *addr = (unsigned char *) page;
46 unsigned char *end = addr + PAGE_SIZE;
47
48 /*
49 * JDCXXX - This should be bottlenecked by the write buffer, but these
50 * things tend to be mildly unpredictable...should check this on the
51 * performance model
52 *
53 * We prefetch 4 lines ahead. We're also "cheating" slightly here...
54 * since we know we're on an SB1, we force the assembler to take
55 * 64-bit operands to speed things up
56 */
57 __asm__ __volatile__(
58 " .set push \n"
59 " .set mips4 \n"
60 " .set noreorder \n"
61#ifdef CONFIG_CPU_HAS_PREFETCH
62 " daddiu %0, %0, 128 \n"
63 " pref " SB1_PREF_STORE_STREAMED_HINT ", -128(%0) \n" /* Prefetch the first 4 lines */
64 " pref " SB1_PREF_STORE_STREAMED_HINT ", -96(%0) \n"
65 " pref " SB1_PREF_STORE_STREAMED_HINT ", -64(%0) \n"
66 " pref " SB1_PREF_STORE_STREAMED_HINT ", -32(%0) \n"
67 "1: sd $0, -128(%0) \n" /* Throw out a cacheline of 0's */
68 " sd $0, -120(%0) \n"
69 " sd $0, -112(%0) \n"
70 " sd $0, -104(%0) \n"
71 " daddiu %0, %0, 32 \n"
72 " bnel %0, %1, 1b \n"
73 " pref " SB1_PREF_STORE_STREAMED_HINT ", -32(%0) \n"
74 " daddiu %0, %0, -128 \n"
75#endif
76 " sd $0, 0(%0) \n" /* Throw out a cacheline of 0's */
77 "1: sd $0, 8(%0) \n"
78 " sd $0, 16(%0) \n"
79 " sd $0, 24(%0) \n"
80 " daddiu %0, %0, 32 \n"
81 " bnel %0, %1, 1b \n"
82 " sd $0, 0(%0) \n"
83 " .set pop \n"
84 : "+r" (addr)
85 : "r" (end)
86 : "memory");
87}
88
89static inline void copy_page_cpu(void *to, void *from)
90{
91 unsigned char *src = (unsigned char *)from;
92 unsigned char *dst = (unsigned char *)to;
93 unsigned char *end = src + PAGE_SIZE;
94
95 /*
96 * The pref's used here are using "streaming" hints, which cause the
97 * copied data to be kicked out of the cache sooner. A page copy often
98 * ends up copying a lot more data than is commonly used, so this seems
99 * to make sense in terms of reducing cache pollution, but I've no real
100 * performance data to back this up
101 */
102 __asm__ __volatile__(
103 " .set push \n"
104 " .set mips4 \n"
105 " .set noreorder \n"
106#ifdef CONFIG_CPU_HAS_PREFETCH
107 " daddiu %0, %0, 128 \n"
108 " daddiu %1, %1, 128 \n"
109 " pref " SB1_PREF_LOAD_STREAMED_HINT ", -128(%0)\n" /* Prefetch the first 4 lines */
110 " pref " SB1_PREF_STORE_STREAMED_HINT ", -128(%1)\n"
111 " pref " SB1_PREF_LOAD_STREAMED_HINT ", -96(%0)\n"
112 " pref " SB1_PREF_STORE_STREAMED_HINT ", -96(%1)\n"
113 " pref " SB1_PREF_LOAD_STREAMED_HINT ", -64(%0)\n"
114 " pref " SB1_PREF_STORE_STREAMED_HINT ", -64(%1)\n"
115 " pref " SB1_PREF_LOAD_STREAMED_HINT ", -32(%0)\n"
116 "1: pref " SB1_PREF_STORE_STREAMED_HINT ", -32(%1)\n"
117# ifdef CONFIG_MIPS64
118 " ld $8, -128(%0) \n" /* Block copy a cacheline */
119 " ld $9, -120(%0) \n"
120 " ld $10, -112(%0) \n"
121 " ld $11, -104(%0) \n"
122 " sd $8, -128(%1) \n"
123 " sd $9, -120(%1) \n"
124 " sd $10, -112(%1) \n"
125 " sd $11, -104(%1) \n"
126# else
127 " lw $2, -128(%0) \n" /* Block copy a cacheline */
128 " lw $3, -124(%0) \n"
129 " lw $6, -120(%0) \n"
130 " lw $7, -116(%0) \n"
131 " lw $8, -112(%0) \n"
132 " lw $9, -108(%0) \n"
133 " lw $10, -104(%0) \n"
134 " lw $11, -100(%0) \n"
135 " sw $2, -128(%1) \n"
136 " sw $3, -124(%1) \n"
137 " sw $6, -120(%1) \n"
138 " sw $7, -116(%1) \n"
139 " sw $8, -112(%1) \n"
140 " sw $9, -108(%1) \n"
141 " sw $10, -104(%1) \n"
142 " sw $11, -100(%1) \n"
143# endif
144 " daddiu %0, %0, 32 \n"
145 " daddiu %1, %1, 32 \n"
146 " bnel %0, %2, 1b \n"
147 " pref " SB1_PREF_LOAD_STREAMED_HINT ", -32(%0)\n"
148 " daddiu %0, %0, -128 \n"
149 " daddiu %1, %1, -128 \n"
150#endif
151#ifdef CONFIG_MIPS64
152 " ld $8, 0(%0) \n" /* Block copy a cacheline */
153 "1: ld $9, 8(%0) \n"
154 " ld $10, 16(%0) \n"
155 " ld $11, 24(%0) \n"
156 " sd $8, 0(%1) \n"
157 " sd $9, 8(%1) \n"
158 " sd $10, 16(%1) \n"
159 " sd $11, 24(%1) \n"
160#else
161 " lw $2, 0(%0) \n" /* Block copy a cacheline */
162 "1: lw $3, 4(%0) \n"
163 " lw $6, 8(%0) \n"
164 " lw $7, 12(%0) \n"
165 " lw $8, 16(%0) \n"
166 " lw $9, 20(%0) \n"
167 " lw $10, 24(%0) \n"
168 " lw $11, 28(%0) \n"
169 " sw $2, 0(%1) \n"
170 " sw $3, 4(%1) \n"
171 " sw $6, 8(%1) \n"
172 " sw $7, 12(%1) \n"
173 " sw $8, 16(%1) \n"
174 " sw $9, 20(%1) \n"
175 " sw $10, 24(%1) \n"
176 " sw $11, 28(%1) \n"
177#endif
178 " daddiu %0, %0, 32 \n"
179 " daddiu %1, %1, 32 \n"
180 " bnel %0, %2, 1b \n"
181#ifdef CONFIG_MIPS64
182 " ld $8, 0(%0) \n"
183#else
184 " lw $2, 0(%0) \n"
185#endif
186 " .set pop \n"
187 : "+r" (src), "+r" (dst)
188 : "r" (end)
189#ifdef CONFIG_MIPS64
190 : "$8","$9","$10","$11","memory");
191#else
192 : "$2","$3","$6","$7","$8","$9","$10","$11","memory");
193#endif
194}
195
196
197#ifdef CONFIG_SIBYTE_DMA_PAGEOPS
198
199/*
200 * Pad descriptors to cacheline, since each is exclusively owned by a
201 * particular CPU.
202 */
203typedef struct dmadscr_s {
204 u64 dscr_a;
205 u64 dscr_b;
206 u64 pad_a;
207 u64 pad_b;
208} dmadscr_t;
209
210static dmadscr_t page_descr[NR_CPUS] __attribute__((aligned(SMP_CACHE_BYTES)));
211
212void sb1_dma_init(void)
213{
214 int cpu = smp_processor_id();
215 u64 base_val = CPHYSADDR(&page_descr[cpu]) | V_DM_DSCR_BASE_RINGSZ(1);
216
217 bus_writeq(base_val,
218 (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
219 bus_writeq(base_val | M_DM_DSCR_BASE_RESET,
220 (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
221 bus_writeq(base_val | M_DM_DSCR_BASE_ENABL,
222 (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
223}
224
225void clear_page(void *page)
226{
227 int cpu = smp_processor_id();
228
229 /* if the page is above Kseg0, use old way */
230 if ((long)KSEGX(page) != (long)CKSEG0)
231 return clear_page_cpu(page);
232
233 page_descr[cpu].dscr_a = CPHYSADDR(page) | M_DM_DSCRA_ZERO_MEM | M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
234 page_descr[cpu].dscr_b = V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
235 bus_writeq(1, (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
236
237 /*
238 * Don't really want to do it this way, but there's no
239 * reliable way to delay completion detection.
240 */
241 while (!(bus_readq((void *)(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)) &
242 M_DM_DSCR_BASE_INTERRUPT))))
243 ;
244 bus_readq((void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
245}
246
247void copy_page(void *to, void *from)
248{
249 unsigned long from_phys = CPHYSADDR(from);
250 unsigned long to_phys = CPHYSADDR(to);
251 int cpu = smp_processor_id();
252
253 /* if either page is above Kseg0, use old way */
254 if ((long)KSEGX(to) != (long)CKSEG0
255 || (long)KSEGX(from) != (long)CKSEG0)
256 return copy_page_cpu(to, from);
257
258 page_descr[cpu].dscr_a = CPHYSADDR(to_phys) | M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
259 page_descr[cpu].dscr_b = CPHYSADDR(from_phys) | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
260 bus_writeq(1, (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
261
262 /*
263 * Don't really want to do it this way, but there's no
264 * reliable way to delay completion detection.
265 */
266 while (!(bus_readq((void *)(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)) &
267 M_DM_DSCR_BASE_INTERRUPT))))
268 ;
269 bus_readq((void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
270}
271
272#else /* !CONFIG_SIBYTE_DMA_PAGEOPS */
273
274void clear_page(void *page)
275{
276 return clear_page_cpu(page);
277}
278
279void copy_page(void *to, void *from)
280{
281 return copy_page_cpu(to, from);
282}
283
284#endif /* !CONFIG_SIBYTE_DMA_PAGEOPS */
285
286EXPORT_SYMBOL(clear_page);
287EXPORT_SYMBOL(copy_page);
diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c
new file mode 100644
index 000000000000..4f07f81e8500
--- /dev/null
+++ b/arch/mips/mm/pgtable-32.c
@@ -0,0 +1,97 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003 by Ralf Baechle
7 */
8#include <linux/config.h>
9#include <linux/init.h>
10#include <linux/mm.h>
11#include <linux/bootmem.h>
12#include <linux/highmem.h>
13#include <asm/pgtable.h>
14
15void pgd_init(unsigned long page)
16{
17 unsigned long *p = (unsigned long *) page;
18 int i;
19
20 for (i = 0; i < USER_PTRS_PER_PGD; i+=8) {
21 p[i + 0] = (unsigned long) invalid_pte_table;
22 p[i + 1] = (unsigned long) invalid_pte_table;
23 p[i + 2] = (unsigned long) invalid_pte_table;
24 p[i + 3] = (unsigned long) invalid_pte_table;
25 p[i + 4] = (unsigned long) invalid_pte_table;
26 p[i + 5] = (unsigned long) invalid_pte_table;
27 p[i + 6] = (unsigned long) invalid_pte_table;
28 p[i + 7] = (unsigned long) invalid_pte_table;
29 }
30}
31
32#ifdef CONFIG_HIGHMEM
33static void __init fixrange_init (unsigned long start, unsigned long end,
34 pgd_t *pgd_base)
35{
36 pgd_t *pgd;
37 pmd_t *pmd;
38 pte_t *pte;
39 int i, j;
40 unsigned long vaddr;
41
42 vaddr = start;
43 i = __pgd_offset(vaddr);
44 j = __pmd_offset(vaddr);
45 pgd = pgd_base + i;
46
47 for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
48 pmd = (pmd_t *)pgd;
49 for (; (j < PTRS_PER_PMD) && (vaddr != end); pmd++, j++) {
50 if (pmd_none(*pmd)) {
51 pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
52 set_pmd(pmd, __pmd((unsigned long)pte));
53 if (pte != pte_offset_kernel(pmd, 0))
54 BUG();
55 }
56 vaddr += PMD_SIZE;
57 }
58 j = 0;
59 }
60}
61#endif
62
63void __init pagetable_init(void)
64{
65#ifdef CONFIG_HIGHMEM
66 unsigned long vaddr;
67 pgd_t *pgd, *pgd_base;
68 pmd_t *pmd;
69 pte_t *pte;
70#endif
71
72 /* Initialize the entire pgd. */
73 pgd_init((unsigned long)swapper_pg_dir);
74 pgd_init((unsigned long)swapper_pg_dir
75 + sizeof(pgd_t) * USER_PTRS_PER_PGD);
76
77#ifdef CONFIG_HIGHMEM
78 pgd_base = swapper_pg_dir;
79
80 /*
81 * Fixed mappings:
82 */
83 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
84 fixrange_init(vaddr, 0, pgd_base);
85
86 /*
87 * Permanent kmaps:
88 */
89 vaddr = PKMAP_BASE;
90 fixrange_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
91
92 pgd = swapper_pg_dir + __pgd_offset(vaddr);
93 pmd = pmd_offset(pgd, vaddr);
94 pte = pte_offset_kernel(pmd, vaddr);
95 pkmap_page_table = pte;
96#endif
97}
diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c
new file mode 100644
index 000000000000..44b5e97fff65
--- /dev/null
+++ b/arch/mips/mm/pgtable-64.c
@@ -0,0 +1,58 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999, 2000 by Silicon Graphics
7 * Copyright (C) 2003 by Ralf Baechle
8 */
9#include <linux/init.h>
10#include <linux/mm.h>
11#include <asm/pgtable.h>
12
13void pgd_init(unsigned long page)
14{
15 unsigned long *p, *end;
16
17 p = (unsigned long *) page;
18 end = p + PTRS_PER_PGD;
19
20 while (p < end) {
21 p[0] = (unsigned long) invalid_pmd_table;
22 p[1] = (unsigned long) invalid_pmd_table;
23 p[2] = (unsigned long) invalid_pmd_table;
24 p[3] = (unsigned long) invalid_pmd_table;
25 p[4] = (unsigned long) invalid_pmd_table;
26 p[5] = (unsigned long) invalid_pmd_table;
27 p[6] = (unsigned long) invalid_pmd_table;
28 p[7] = (unsigned long) invalid_pmd_table;
29 p += 8;
30 }
31}
32
33void pmd_init(unsigned long addr, unsigned long pagetable)
34{
35 unsigned long *p, *end;
36
37 p = (unsigned long *) addr;
38 end = p + PTRS_PER_PMD;
39
40 while (p < end) {
41 p[0] = (unsigned long)pagetable;
42 p[1] = (unsigned long)pagetable;
43 p[2] = (unsigned long)pagetable;
44 p[3] = (unsigned long)pagetable;
45 p[4] = (unsigned long)pagetable;
46 p[5] = (unsigned long)pagetable;
47 p[6] = (unsigned long)pagetable;
48 p[7] = (unsigned long)pagetable;
49 p += 8;
50 }
51}
52
53void __init pagetable_init(void)
54{
55 /* Initialize the entire pgd. */
56 pgd_init((unsigned long)swapper_pg_dir);
57 pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
58}
diff --git a/arch/mips/mm/pgtable.c b/arch/mips/mm/pgtable.c
new file mode 100644
index 000000000000..3b88fdeef329
--- /dev/null
+++ b/arch/mips/mm/pgtable.c
@@ -0,0 +1,36 @@
1#include <linux/config.h>
2#include <linux/kernel.h>
3#include <linux/mm.h>
4#include <linux/swap.h>
5
6void show_mem(void)
7{
8#ifndef CONFIG_DISCONTIGMEM /* XXX(hch): later.. */
9 int pfn, total = 0, reserved = 0;
10 int shared = 0, cached = 0;
11 int highmem = 0;
12 struct page *page;
13
14 printk("Mem-info:\n");
15 show_free_areas();
16 printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
17 pfn = max_mapnr;
18 while (pfn-- > 0) {
19 page = pfn_to_page(pfn);
20 total++;
21 if (PageHighMem(page))
22 highmem++;
23 if (PageReserved(page))
24 reserved++;
25 else if (PageSwapCache(page))
26 cached++;
27 else if (page_count(page))
28 shared += page_count(page) - 1;
29 }
30 printk("%d pages of RAM\n", total);
31 printk("%d pages of HIGHMEM\n",highmem);
32 printk("%d reserved pages\n",reserved);
33 printk("%d pages shared\n",shared);
34 printk("%d pages swap cached\n",cached);
35#endif
36}
diff --git a/arch/mips/mm/sc-ip22.c b/arch/mips/mm/sc-ip22.c
new file mode 100644
index 000000000000..d236cf8b7374
--- /dev/null
+++ b/arch/mips/mm/sc-ip22.c
@@ -0,0 +1,177 @@
1/*
2 * sc-ip22.c: Indy cache management functions.
3 *
4 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org),
5 * derived from r4xx0.c by David S. Miller (dm@engr.sgi.com).
6 */
7#include <linux/init.h>
8#include <linux/kernel.h>
9#include <linux/sched.h>
10#include <linux/mm.h>
11
12#include <asm/bcache.h>
13#include <asm/page.h>
14#include <asm/pgtable.h>
15#include <asm/system.h>
16#include <asm/bootinfo.h>
17#include <asm/sgi/ip22.h>
18#include <asm/sgi/mc.h>
19
20/* Secondary cache size in bytes, if present. */
21static unsigned long scache_size;
22
23#undef DEBUG_CACHE
24
25#define SC_SIZE 0x00080000
26#define SC_LINE 32
27#define CI_MASK (SC_SIZE - SC_LINE)
28#define SC_INDEX(n) ((n) & CI_MASK)
29
30static inline void indy_sc_wipe(unsigned long first, unsigned long last)
31{
32 unsigned long tmp;
33
34 __asm__ __volatile__(
35 ".set\tpush\t\t\t# indy_sc_wipe\n\t"
36 ".set\tnoreorder\n\t"
37 ".set\tmips3\n\t"
38 ".set\tnoat\n\t"
39 "mfc0\t%2, $12\n\t"
40 "li\t$1, 0x80\t\t\t# Go 64 bit\n\t"
41 "mtc0\t$1, $12\n\t"
42
43 "dli\t$1, 0x9000000080000000\n\t"
44 "or\t%0, $1\t\t\t# first line to flush\n\t"
45 "or\t%1, $1\t\t\t# last line to flush\n\t"
46 ".set\tat\n\t"
47
48 "1:\tsw\t$0, 0(%0)\n\t"
49 "bne\t%0, %1, 1b\n\t"
50 " daddu\t%0, 32\n\t"
51
52 "mtc0\t%2, $12\t\t\t# Back to 32 bit\n\t"
53 "nop; nop; nop; nop;\n\t"
54 ".set\tpop"
55 : "=r" (first), "=r" (last), "=&r" (tmp)
56 : "0" (first), "1" (last));
57}
58
59static void indy_sc_wback_invalidate(unsigned long addr, unsigned long size)
60{
61 unsigned long first_line, last_line;
62 unsigned long flags;
63
64#ifdef DEBUG_CACHE
65 printk("indy_sc_wback_invalidate[%08lx,%08lx]", addr, size);
66#endif
67
68 /* Catch bad driver code */
69 BUG_ON(size == 0);
70
71 /* Which lines to flush? */
72 first_line = SC_INDEX(addr);
73 last_line = SC_INDEX(addr + size - 1);
74
75 local_irq_save(flags);
76 if (first_line <= last_line) {
77 indy_sc_wipe(first_line, last_line);
78 goto out;
79 }
80
81 indy_sc_wipe(first_line, SC_SIZE - SC_LINE);
82 indy_sc_wipe(0, last_line);
83out:
84 local_irq_restore(flags);
85}
86
87static void indy_sc_enable(void)
88{
89 unsigned long addr, tmp1, tmp2;
90
91 /* This is really cool... */
92#ifdef DEBUG_CACHE
93 printk("Enabling R4600 SCACHE\n");
94#endif
95 __asm__ __volatile__(
96 ".set\tpush\n\t"
97 ".set\tnoreorder\n\t"
98 ".set\tmips3\n\t"
99 "mfc0\t%2, $12\n\t"
100 "nop; nop; nop; nop;\n\t"
101 "li\t%1, 0x80\n\t"
102 "mtc0\t%1, $12\n\t"
103 "nop; nop; nop; nop;\n\t"
104 "li\t%0, 0x1\n\t"
105 "dsll\t%0, 31\n\t"
106 "lui\t%1, 0x9000\n\t"
107 "dsll32\t%1, 0\n\t"
108 "or\t%0, %1, %0\n\t"
109 "sb\t$0, 0(%0)\n\t"
110 "mtc0\t$0, $12\n\t"
111 "nop; nop; nop; nop;\n\t"
112 "mtc0\t%2, $12\n\t"
113 "nop; nop; nop; nop;\n\t"
114 ".set\tpop"
115 : "=r" (tmp1), "=r" (tmp2), "=r" (addr));
116}
117
118static void indy_sc_disable(void)
119{
120 unsigned long tmp1, tmp2, tmp3;
121
122#ifdef DEBUG_CACHE
123 printk("Disabling R4600 SCACHE\n");
124#endif
125 __asm__ __volatile__(
126 ".set\tpush\n\t"
127 ".set\tnoreorder\n\t"
128 ".set\tmips3\n\t"
129 "li\t%0, 0x1\n\t"
130 "dsll\t%0, 31\n\t"
131 "lui\t%1, 0x9000\n\t"
132 "dsll32\t%1, 0\n\t"
133 "or\t%0, %1, %0\n\t"
134 "mfc0\t%2, $12\n\t"
135 "nop; nop; nop; nop\n\t"
136 "li\t%1, 0x80\n\t"
137 "mtc0\t%1, $12\n\t"
138 "nop; nop; nop; nop\n\t"
139 "sh\t$0, 0(%0)\n\t"
140 "mtc0\t$0, $12\n\t"
141 "nop; nop; nop; nop\n\t"
142 "mtc0\t%2, $12\n\t"
143 "nop; nop; nop; nop\n\t"
144 ".set\tpop"
145 : "=r" (tmp1), "=r" (tmp2), "=r" (tmp3));
146}
147
148static inline int __init indy_sc_probe(void)
149{
150 unsigned int size = ip22_eeprom_read(&sgimc->eeprom, 17);
151 if (size == 0)
152 return 0;
153
154 size <<= PAGE_SHIFT;
155 printk(KERN_INFO "R4600/R5000 SCACHE size %dK, linesize 32 bytes.\n",
156 size >> 10);
157 scache_size = size;
158
159 return 1;
160}
161
162/* XXX Check with wje if the Indy caches can differenciate between
163 writeback + invalidate and just invalidate. */
164struct bcache_ops indy_sc_ops = {
165 .bc_enable = indy_sc_enable,
166 .bc_disable = indy_sc_disable,
167 .bc_wback_inv = indy_sc_wback_invalidate,
168 .bc_inv = indy_sc_wback_invalidate
169};
170
171void __init indy_sc_init(void)
172{
173 if (indy_sc_probe()) {
174 indy_sc_enable();
175 bcops = &indy_sc_ops;
176 }
177}
diff --git a/arch/mips/mm/sc-r5k.c b/arch/mips/mm/sc-r5k.c
new file mode 100644
index 000000000000..d35b6c1103a3
--- /dev/null
+++ b/arch/mips/mm/sc-r5k.c
@@ -0,0 +1,108 @@
1/*
2 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org),
3 * derived from r4xx0.c by David S. Miller (dm@engr.sgi.com).
4 */
5#include <linux/init.h>
6#include <linux/kernel.h>
7#include <linux/sched.h>
8#include <linux/mm.h>
9
10#include <asm/mipsregs.h>
11#include <asm/bcache.h>
12#include <asm/cacheops.h>
13#include <asm/page.h>
14#include <asm/pgtable.h>
15#include <asm/system.h>
16#include <asm/mmu_context.h>
17#include <asm/r4kcache.h>
18
19/* Secondary cache size in bytes, if present. */
20static unsigned long scache_size;
21
22#define SC_LINE 32
23#define SC_PAGE (128*SC_LINE)
24
25static inline void blast_r5000_scache(void)
26{
27 unsigned long start = INDEX_BASE;
28 unsigned long end = start + scache_size;
29
30 while(start < end) {
31 cache_op(R5K_Page_Invalidate_S, start);
32 start += SC_PAGE;
33 }
34}
35
36static void r5k_dma_cache_inv_sc(unsigned long addr, unsigned long size)
37{
38 unsigned long end, a;
39
40 /* Catch bad driver code */
41 BUG_ON(size == 0);
42
43 if (size >= scache_size) {
44 blast_r5000_scache();
45 return;
46 }
47
48 /* On the R5000 secondary cache we cannot
49 * invalidate less than a page at a time.
50 * The secondary cache is physically indexed, write-through.
51 */
52 a = addr & ~(SC_PAGE - 1);
53 end = (addr + size - 1) & ~(SC_PAGE - 1);
54 while (a <= end) {
55 cache_op(R5K_Page_Invalidate_S, a);
56 a += SC_PAGE;
57 }
58}
59
60static void r5k_sc_enable(void)
61{
62 unsigned long flags;
63
64 local_irq_save(flags);
65 set_c0_config(R5K_CONF_SE);
66 blast_r5000_scache();
67 local_irq_restore(flags);
68}
69
70static void r5k_sc_disable(void)
71{
72 unsigned long flags;
73
74 local_irq_save(flags);
75 blast_r5000_scache();
76 clear_c0_config(R5K_CONF_SE);
77 local_irq_restore(flags);
78}
79
80static inline int __init r5k_sc_probe(void)
81{
82 unsigned long config = read_c0_config();
83
84 if (config & CONF_SC)
85 return(0);
86
87 scache_size = (512 * 1024) << ((config & R5K_CONF_SS) >> 20);
88
89 printk("R5000 SCACHE size %ldkB, linesize 32 bytes.\n",
90 scache_size >> 10);
91
92 return 1;
93}
94
95static struct bcache_ops r5k_sc_ops = {
96 .bc_enable = r5k_sc_enable,
97 .bc_disable = r5k_sc_disable,
98 .bc_wback_inv = r5k_dma_cache_inv_sc,
99 .bc_inv = r5k_dma_cache_inv_sc
100};
101
102void __init r5k_sc_init(void)
103{
104 if (r5k_sc_probe()) {
105 r5k_sc_enable();
106 bcops = &r5k_sc_ops;
107 }
108}
diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c
new file mode 100644
index 000000000000..4e92f931aaba
--- /dev/null
+++ b/arch/mips/mm/sc-rm7k.c
@@ -0,0 +1,193 @@
1/*
2 * sc-rm7k.c: RM7000 cache management functions.
3 *
4 * Copyright (C) 1997, 2001, 2003, 2004 Ralf Baechle (ralf@linux-mips.org)
5 */
6
7#undef DEBUG
8
9#include <linux/init.h>
10#include <linux/kernel.h>
11#include <linux/mm.h>
12
13#include <asm/addrspace.h>
14#include <asm/bcache.h>
15#include <asm/cacheops.h>
16#include <asm/mipsregs.h>
17#include <asm/processor.h>
18
19/* Primary cache parameters. */
20#define sc_lsize 32
21#define tc_pagesize (32*128)
22
23/* Secondary cache parameters. */
24#define scache_size (256*1024) /* Fixed to 256KiB on RM7000 */
25
26extern unsigned long icache_way_size, dcache_way_size;
27
28#include <asm/r4kcache.h>
29
30int rm7k_tcache_enabled;
31
32/*
33 * Writeback and invalidate the primary cache dcache before DMA.
34 * (XXX These need to be fixed ...)
35 */
36static void rm7k_sc_wback_inv(unsigned long addr, unsigned long size)
37{
38 unsigned long end, a;
39
40 pr_debug("rm7k_sc_wback_inv[%08lx,%08lx]", addr, size);
41
42 /* Catch bad driver code */
43 BUG_ON(size == 0);
44
45 a = addr & ~(sc_lsize - 1);
46 end = (addr + size - 1) & ~(sc_lsize - 1);
47 while (1) {
48 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
49 if (a == end)
50 break;
51 a += sc_lsize;
52 }
53
54 if (!rm7k_tcache_enabled)
55 return;
56
57 a = addr & ~(tc_pagesize - 1);
58 end = (addr + size - 1) & ~(tc_pagesize - 1);
59 while(1) {
60 invalidate_tcache_page(a); /* Page_Invalidate_T */
61 if (a == end)
62 break;
63 a += tc_pagesize;
64 }
65}
66
67static void rm7k_sc_inv(unsigned long addr, unsigned long size)
68{
69 unsigned long end, a;
70
71 pr_debug("rm7k_sc_inv[%08lx,%08lx]", addr, size);
72
73 /* Catch bad driver code */
74 BUG_ON(size == 0);
75
76 a = addr & ~(sc_lsize - 1);
77 end = (addr + size - 1) & ~(sc_lsize - 1);
78 while (1) {
79 invalidate_scache_line(a); /* Hit_Invalidate_SD */
80 if (a == end)
81 break;
82 a += sc_lsize;
83 }
84
85 if (!rm7k_tcache_enabled)
86 return;
87
88 a = addr & ~(tc_pagesize - 1);
89 end = (addr + size - 1) & ~(tc_pagesize - 1);
90 while(1) {
91 invalidate_tcache_page(a); /* Page_Invalidate_T */
92 if (a == end)
93 break;
94 a += tc_pagesize;
95 }
96}
97
98/*
99 * This function is executed in the uncached segment CKSEG1.
100 * It must not touch the stack, because the stack pointer still points
101 * into CKSEG0.
102 *
103 * Three options:
104 * - Write it in assembly and guarantee that we don't use the stack.
105 * - Disable caching for CKSEG0 before calling it.
106 * - Pray that GCC doesn't randomly start using the stack.
107 *
108 * This being Linux, we obviously take the least sane of those options -
109 * following DaveM's lead in c-r4k.c
110 *
111 * It seems we get our kicks from relying on unguaranteed behaviour in GCC
112 */
113static __init void __rm7k_sc_enable(void)
114{
115 int i;
116
117 set_c0_config(1 << 3); /* CONF_SE */
118
119 write_c0_taglo(0);
120 write_c0_taghi(0);
121
122 for (i = 0; i < scache_size; i += sc_lsize) {
123 __asm__ __volatile__ (
124 ".set noreorder\n\t"
125 ".set mips3\n\t"
126 "cache %1, (%0)\n\t"
127 ".set mips0\n\t"
128 ".set reorder"
129 :
130 : "r" (KSEG0ADDR(i)), "i" (Index_Store_Tag_SD));
131 }
132}
133
134static __init void rm7k_sc_enable(void)
135{
136 void (*func)(void) = (void *) KSEG1ADDR(&__rm7k_sc_enable);
137
138 if (read_c0_config() & 0x08) /* CONF_SE */
139 return;
140
141 printk(KERN_INFO "Enabling secondary cache...");
142 func();
143}
144
145static void rm7k_sc_disable(void)
146{
147 clear_c0_config(1<<3); /* CONF_SE */
148}
149
150struct bcache_ops rm7k_sc_ops = {
151 .bc_enable = rm7k_sc_enable,
152 .bc_disable = rm7k_sc_disable,
153 .bc_wback_inv = rm7k_sc_wback_inv,
154 .bc_inv = rm7k_sc_inv
155};
156
157void __init rm7k_sc_init(void)
158{
159 unsigned int config = read_c0_config();
160
161 if ((config >> 31) & 1) /* Bit 31 set -> no S-Cache */
162 return;
163
164 printk(KERN_INFO "Secondary cache size %dK, linesize %d bytes.\n",
165 (scache_size >> 10), sc_lsize);
166
167 if (!((config >> 3) & 1)) /* CONF_SE */
168 rm7k_sc_enable();
169
170 /*
171 * While we're at it let's deal with the tertiary cache.
172 */
173 if (!((config >> 17) & 1)) {
174
175 /*
176 * We can't enable the L3 cache yet. There may be board-specific
177 * magic necessary to turn it on, and blindly asking the CPU to
178 * start using it would may give cache errors.
179 *
180 * Also, board-specific knowledge may allow us to use the
181 * CACHE Flash_Invalidate_T instruction if the tag RAM supports
182 * it, and may specify the size of the L3 cache so we don't have
183 * to probe it.
184 */
185 printk(KERN_INFO "Tertiary cache present, %s enabled\n",
186 config&(1<<12) ? "already" : "not (yet)");
187
188 if ((config >> 12) & 1)
189 rm7k_tcache_enabled = 1;
190 }
191
192 bcops = &rm7k_sc_ops;
193}
diff --git a/arch/mips/mm/tlb-andes.c b/arch/mips/mm/tlb-andes.c
new file mode 100644
index 000000000000..167e08e9661a
--- /dev/null
+++ b/arch/mips/mm/tlb-andes.c
@@ -0,0 +1,257 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1997, 1998, 1999 Ralf Baechle (ralf@gnu.org)
7 * Copyright (C) 1999 Silicon Graphics, Inc.
8 * Copyright (C) 2000 Kanoj Sarcar (kanoj@sgi.com)
9 */
10#include <linux/init.h>
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/mm.h>
14#include <asm/page.h>
15#include <asm/pgtable.h>
16#include <asm/system.h>
17#include <asm/mmu_context.h>
18
19extern void build_tlb_refill_handler(void);
20
21#define NTLB_ENTRIES 64
22#define NTLB_ENTRIES_HALF 32
23
24void local_flush_tlb_all(void)
25{
26 unsigned long flags;
27 unsigned long old_ctx;
28 unsigned long entry;
29
30 local_irq_save(flags);
31 /* Save old context and create impossible VPN2 value */
32 old_ctx = read_c0_entryhi() & ASID_MASK;
33 write_c0_entryhi(CKSEG0);
34 write_c0_entrylo0(0);
35 write_c0_entrylo1(0);
36
37 entry = read_c0_wired();
38
39 /* Blast 'em all away. */
40 while (entry < NTLB_ENTRIES) {
41 write_c0_index(entry);
42 tlb_write_indexed();
43 entry++;
44 }
45 write_c0_entryhi(old_ctx);
46 local_irq_restore(flags);
47}
48
49void local_flush_tlb_mm(struct mm_struct *mm)
50{
51 int cpu = smp_processor_id();
52 if (cpu_context(cpu, mm) != 0) {
53 drop_mmu_context(mm,cpu);
54 }
55}
56
57void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
58 unsigned long end)
59{
60 struct mm_struct *mm = vma->vm_mm;
61 int cpu = smp_processor_id();
62
63 if (cpu_context(cpu, mm) != 0) {
64 unsigned long flags;
65 int size;
66
67 local_irq_save(flags);
68 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
69 size = (size + 1) >> 1;
70 if (size <= NTLB_ENTRIES_HALF) {
71 int oldpid = (read_c0_entryhi() & ASID_MASK);
72 int newpid = (cpu_context(smp_processor_id(), mm)
73 & ASID_MASK);
74
75 start &= (PAGE_MASK << 1);
76 end += ((PAGE_SIZE << 1) - 1);
77 end &= (PAGE_MASK << 1);
78 while(start < end) {
79 int idx;
80
81 write_c0_entryhi(start | newpid);
82 start += (PAGE_SIZE << 1);
83 tlb_probe();
84 idx = read_c0_index();
85 write_c0_entrylo0(0);
86 write_c0_entrylo1(0);
87 write_c0_entryhi(CKSEG0);
88 if(idx < 0)
89 continue;
90 tlb_write_indexed();
91 }
92 write_c0_entryhi(oldpid);
93 } else {
94 drop_mmu_context(mm, cpu);
95 }
96 local_irq_restore(flags);
97 }
98}
99
100void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
101{
102 unsigned long flags;
103 int size;
104
105 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
106 size = (size + 1) >> 1;
107
108 local_irq_save(flags);
109 if (size <= NTLB_ENTRIES_HALF) {
110 int pid = read_c0_entryhi();
111
112 start &= (PAGE_MASK << 1);
113 end += ((PAGE_SIZE << 1) - 1);
114 end &= (PAGE_MASK << 1);
115
116 while (start < end) {
117 int idx;
118
119 write_c0_entryhi(start);
120 start += (PAGE_SIZE << 1);
121 tlb_probe();
122 idx = read_c0_index();
123 write_c0_entrylo0(0);
124 write_c0_entrylo1(0);
125 write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT+1)));
126 if (idx < 0)
127 continue;
128 tlb_write_indexed();
129 }
130 write_c0_entryhi(pid);
131 } else {
132 local_flush_tlb_all();
133 }
134 local_irq_restore(flags);
135}
136
137void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
138{
139 if (cpu_context(smp_processor_id(), vma->vm_mm) != 0) {
140 unsigned long flags;
141 int oldpid, newpid, idx;
142
143 newpid = (cpu_context(smp_processor_id(), vma->vm_mm) &
144 ASID_MASK);
145 page &= (PAGE_MASK << 1);
146 local_irq_save(flags);
147 oldpid = (read_c0_entryhi() & ASID_MASK);
148 write_c0_entryhi(page | newpid);
149 tlb_probe();
150 idx = read_c0_index();
151 write_c0_entrylo0(0);
152 write_c0_entrylo1(0);
153 write_c0_entryhi(CKSEG0);
154 if (idx < 0)
155 goto finish;
156 tlb_write_indexed();
157
158 finish:
159 write_c0_entryhi(oldpid);
160 local_irq_restore(flags);
161 }
162}
163
164/*
165 * This one is only used for pages with the global bit set so we don't care
166 * much about the ASID.
167 */
168void local_flush_tlb_one(unsigned long page)
169{
170 unsigned long flags;
171 int oldpid, idx;
172
173 local_irq_save(flags);
174 page &= (PAGE_MASK << 1);
175 oldpid = read_c0_entryhi() & 0xff;
176 write_c0_entryhi(page);
177 tlb_probe();
178 idx = read_c0_index();
179 write_c0_entrylo0(0);
180 write_c0_entrylo1(0);
181 if (idx >= 0) {
182 /* Make sure all entries differ. */
183 write_c0_entryhi(CKSEG0+(idx<<(PAGE_SHIFT+1)));
184 tlb_write_indexed();
185 }
186 write_c0_entryhi(oldpid);
187
188 local_irq_restore(flags);
189}
190
191/* XXX Simplify this. On the R10000 writing a TLB entry for an virtual
192 address that already exists will overwrite the old entry and not result
193 in TLB malfunction or TLB shutdown. */
194void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
195{
196 unsigned long flags;
197 pgd_t *pgdp;
198 pmd_t *pmdp;
199 pte_t *ptep;
200 int idx, pid;
201
202 /*
203 * Handle debugger faulting in for debugee.
204 */
205 if (current->active_mm != vma->vm_mm)
206 return;
207
208 pid = read_c0_entryhi() & ASID_MASK;
209
210 if ((pid != (cpu_context(smp_processor_id(), vma->vm_mm) & ASID_MASK))
211 || (cpu_context(smp_processor_id(), vma->vm_mm) == 0)) {
212 printk(KERN_WARNING
213 "%s: Wheee, bogus tlbpid mmpid=%d tlbpid=%d\n",
214 __FUNCTION__, (int) (cpu_context(smp_processor_id(),
215 vma->vm_mm) & ASID_MASK), pid);
216 }
217
218 local_irq_save(flags);
219 address &= (PAGE_MASK << 1);
220 write_c0_entryhi(address | (pid));
221 pgdp = pgd_offset(vma->vm_mm, address);
222 tlb_probe();
223 pmdp = pmd_offset(pgdp, address);
224 idx = read_c0_index();
225 ptep = pte_offset_map(pmdp, address);
226 write_c0_entrylo0(pte_val(*ptep++) >> 6);
227 write_c0_entrylo1(pte_val(*ptep) >> 6);
228 write_c0_entryhi(address | pid);
229 if (idx < 0) {
230 tlb_write_random();
231 } else {
232 tlb_write_indexed();
233 }
234 write_c0_entryhi(pid);
235 local_irq_restore(flags);
236}
237
238void __init tlb_init(void)
239{
240 /*
241 * You should never change this register:
242 * - On R4600 1.7 the tlbp never hits for pages smaller than
243 * the value in the c0_pagemask register.
244 * - The entire mm handling assumes the c0_pagemask register to
245 * be set for 4kb pages.
246 */
247 write_c0_pagemask(PM_4K);
248 write_c0_wired(0);
249 write_c0_framemask(0);
250
251 /* From this point on the ARC firmware is dead. */
252 local_flush_tlb_all();
253
254 /* Did I tell you that ARC SUCKS? */
255
256 build_tlb_refill_handler();
257}
diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c
new file mode 100644
index 000000000000..7948e9a5e372
--- /dev/null
+++ b/arch/mips/mm/tlb-r3k.c
@@ -0,0 +1,289 @@
1/*
2 * r2300.c: R2000 and R3000 specific mmu/cache code.
3 *
4 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
5 *
6 * with a lot of changes to make this thing work for R3000s
7 * Tx39XX R4k style caches added. HK
8 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
9 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
10 * Copyright (C) 2002 Ralf Baechle
11 * Copyright (C) 2002 Maciej W. Rozycki
12 */
13#include <linux/init.h>
14#include <linux/kernel.h>
15#include <linux/sched.h>
16#include <linux/mm.h>
17
18#include <asm/page.h>
19#include <asm/pgtable.h>
20#include <asm/mmu_context.h>
21#include <asm/system.h>
22#include <asm/isadep.h>
23#include <asm/io.h>
24#include <asm/bootinfo.h>
25#include <asm/cpu.h>
26
27#undef DEBUG_TLB
28
29extern void build_tlb_refill_handler(void);
30
31/* CP0 hazard avoidance. */
32#define BARRIER \
33 __asm__ __volatile__( \
34 ".set push\n\t" \
35 ".set noreorder\n\t" \
36 "nop\n\t" \
37 ".set pop\n\t")
38
39int r3k_have_wired_reg; /* should be in cpu_data? */
40
41/* TLB operations. */
42void local_flush_tlb_all(void)
43{
44 unsigned long flags;
45 unsigned long old_ctx;
46 int entry;
47
48#ifdef DEBUG_TLB
49 printk("[tlball]");
50#endif
51
52 local_irq_save(flags);
53 old_ctx = read_c0_entryhi() & ASID_MASK;
54 write_c0_entrylo0(0);
55 entry = r3k_have_wired_reg ? read_c0_wired() : 8;
56 for (; entry < current_cpu_data.tlbsize; entry++) {
57 write_c0_index(entry << 8);
58 write_c0_entryhi((entry | 0x80000) << 12);
59 BARRIER;
60 tlb_write_indexed();
61 }
62 write_c0_entryhi(old_ctx);
63 local_irq_restore(flags);
64}
65
66void local_flush_tlb_mm(struct mm_struct *mm)
67{
68 int cpu = smp_processor_id();
69
70 if (cpu_context(cpu, mm) != 0) {
71#ifdef DEBUG_TLB
72 printk("[tlbmm<%lu>]", (unsigned long)cpu_context(cpu, mm));
73#endif
74 drop_mmu_context(mm, cpu);
75 }
76}
77
78void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
79 unsigned long end)
80{
81 struct mm_struct *mm = vma->vm_mm;
82 int cpu = smp_processor_id();
83
84 if (cpu_context(cpu, mm) != 0) {
85 unsigned long flags;
86 int size;
87
88#ifdef DEBUG_TLB
89 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
90 cpu_context(cpu, mm) & ASID_MASK, start, end);
91#endif
92 local_irq_save(flags);
93 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
94 if (size <= current_cpu_data.tlbsize) {
95 int oldpid = read_c0_entryhi() & ASID_MASK;
96 int newpid = cpu_context(cpu, mm) & ASID_MASK;
97
98 start &= PAGE_MASK;
99 end += PAGE_SIZE - 1;
100 end &= PAGE_MASK;
101 while (start < end) {
102 int idx;
103
104 write_c0_entryhi(start | newpid);
105 start += PAGE_SIZE; /* BARRIER */
106 tlb_probe();
107 idx = read_c0_index();
108 write_c0_entrylo0(0);
109 write_c0_entryhi(KSEG0);
110 if (idx < 0) /* BARRIER */
111 continue;
112 tlb_write_indexed();
113 }
114 write_c0_entryhi(oldpid);
115 } else {
116 drop_mmu_context(mm, cpu);
117 }
118 local_irq_restore(flags);
119 }
120}
121
122void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
123{
124 unsigned long flags;
125 int size;
126
127#ifdef DEBUG_TLB
128 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
129#endif
130 local_irq_save(flags);
131 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
132 if (size <= current_cpu_data.tlbsize) {
133 int pid = read_c0_entryhi();
134
135 start &= PAGE_MASK;
136 end += PAGE_SIZE - 1;
137 end &= PAGE_MASK;
138
139 while (start < end) {
140 int idx;
141
142 write_c0_entryhi(start);
143 start += PAGE_SIZE; /* BARRIER */
144 tlb_probe();
145 idx = read_c0_index();
146 write_c0_entrylo0(0);
147 write_c0_entryhi(KSEG0);
148 if (idx < 0) /* BARRIER */
149 continue;
150 tlb_write_indexed();
151 }
152 write_c0_entryhi(pid);
153 } else {
154 local_flush_tlb_all();
155 }
156 local_irq_restore(flags);
157}
158
159void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
160{
161 int cpu = smp_processor_id();
162
163 if (!vma || cpu_context(cpu, vma->vm_mm) != 0) {
164 unsigned long flags;
165 int oldpid, newpid, idx;
166
167#ifdef DEBUG_TLB
168 printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
169#endif
170 newpid = cpu_context(cpu, vma->vm_mm) & ASID_MASK;
171 page &= PAGE_MASK;
172 local_irq_save(flags);
173 oldpid = read_c0_entryhi() & ASID_MASK;
174 write_c0_entryhi(page | newpid);
175 BARRIER;
176 tlb_probe();
177 idx = read_c0_index();
178 write_c0_entrylo0(0);
179 write_c0_entryhi(KSEG0);
180 if (idx < 0) /* BARRIER */
181 goto finish;
182 tlb_write_indexed();
183
184finish:
185 write_c0_entryhi(oldpid);
186 local_irq_restore(flags);
187 }
188}
189
190void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
191{
192 unsigned long flags;
193 int idx, pid;
194
195 /*
196 * Handle debugger faulting in for debugee.
197 */
198 if (current->active_mm != vma->vm_mm)
199 return;
200
201 pid = read_c0_entryhi() & ASID_MASK;
202
203#ifdef DEBUG_TLB
204 if ((pid != (cpu_context(cpu, vma->vm_mm) & ASID_MASK)) || (cpu_context(cpu, vma->vm_mm) == 0)) {
205 printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
206 (cpu_context(cpu, vma->vm_mm)), pid);
207 }
208#endif
209
210 local_irq_save(flags);
211 address &= PAGE_MASK;
212 write_c0_entryhi(address | pid);
213 BARRIER;
214 tlb_probe();
215 idx = read_c0_index();
216 write_c0_entrylo0(pte_val(pte));
217 write_c0_entryhi(address | pid);
218 if (idx < 0) { /* BARRIER */
219 tlb_write_random();
220 } else {
221 tlb_write_indexed();
222 }
223 write_c0_entryhi(pid);
224 local_irq_restore(flags);
225}
226
227void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
228 unsigned long entryhi, unsigned long pagemask)
229{
230 unsigned long flags;
231 unsigned long old_ctx;
232 static unsigned long wired = 0;
233
234 if (r3k_have_wired_reg) { /* TX39XX */
235 unsigned long old_pagemask;
236 unsigned long w;
237
238#ifdef DEBUG_TLB
239 printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
240 entrylo0, entryhi, pagemask);
241#endif
242
243 local_irq_save(flags);
244 /* Save old context and create impossible VPN2 value */
245 old_ctx = read_c0_entryhi() & ASID_MASK;
246 old_pagemask = read_c0_pagemask();
247 w = read_c0_wired();
248 write_c0_wired(w + 1);
249 if (read_c0_wired() != w + 1) {
250 printk("[tlbwired] No WIRED reg?\n");
251 return;
252 }
253 write_c0_index(w << 8);
254 write_c0_pagemask(pagemask);
255 write_c0_entryhi(entryhi);
256 write_c0_entrylo0(entrylo0);
257 BARRIER;
258 tlb_write_indexed();
259
260 write_c0_entryhi(old_ctx);
261 write_c0_pagemask(old_pagemask);
262 local_flush_tlb_all();
263 local_irq_restore(flags);
264
265 } else if (wired < 8) {
266#ifdef DEBUG_TLB
267 printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
268 entrylo0, entryhi);
269#endif
270
271 local_irq_save(flags);
272 old_ctx = read_c0_entryhi() & ASID_MASK;
273 write_c0_entrylo0(entrylo0);
274 write_c0_entryhi(entryhi);
275 write_c0_index(wired);
276 wired++; /* BARRIER */
277 tlb_write_indexed();
278 write_c0_entryhi(old_ctx);
279 local_flush_tlb_all();
280 local_irq_restore(flags);
281 }
282}
283
284void __init tlb_init(void)
285{
286 local_flush_tlb_all();
287
288 build_tlb_refill_handler();
289}
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
new file mode 100644
index 000000000000..59d38bc05b69
--- /dev/null
+++ b/arch/mips/mm/tlb-r4k.c
@@ -0,0 +1,419 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
8 * Carsten Langgaard, carstenl@mips.com
9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved.
10 */
11#include <linux/config.h>
12#include <linux/init.h>
13#include <linux/sched.h>
14#include <linux/mm.h>
15
16#include <asm/cpu.h>
17#include <asm/bootinfo.h>
18#include <asm/mmu_context.h>
19#include <asm/pgtable.h>
20#include <asm/system.h>
21
22extern void build_tlb_refill_handler(void);
23
24/* CP0 hazard avoidance. */
25#define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
26 "nop; nop; nop; nop; nop; nop;\n\t" \
27 ".set reorder\n\t")
28
29void local_flush_tlb_all(void)
30{
31 unsigned long flags;
32 unsigned long old_ctx;
33 int entry;
34
35 local_irq_save(flags);
36 /* Save old context and create impossible VPN2 value */
37 old_ctx = read_c0_entryhi();
38 write_c0_entrylo0(0);
39 write_c0_entrylo1(0);
40
41 entry = read_c0_wired();
42
43 /* Blast 'em all away. */
44 while (entry < current_cpu_data.tlbsize) {
45 /*
46 * Make sure all entries differ. If they're not different
47 * MIPS32 will take revenge ...
48 */
49 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
50 write_c0_index(entry);
51 mtc0_tlbw_hazard();
52 tlb_write_indexed();
53 entry++;
54 }
55 tlbw_use_hazard();
56 write_c0_entryhi(old_ctx);
57 local_irq_restore(flags);
58}
59
60void local_flush_tlb_mm(struct mm_struct *mm)
61{
62 int cpu = smp_processor_id();
63
64 if (cpu_context(cpu, mm) != 0)
65 drop_mmu_context(mm,cpu);
66}
67
68void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
69 unsigned long end)
70{
71 struct mm_struct *mm = vma->vm_mm;
72 int cpu = smp_processor_id();
73
74 if (cpu_context(cpu, mm) != 0) {
75 unsigned long flags;
76 int size;
77
78 local_irq_save(flags);
79 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
80 size = (size + 1) >> 1;
81 if (size <= current_cpu_data.tlbsize/2) {
82 int oldpid = read_c0_entryhi();
83 int newpid = cpu_asid(cpu, mm);
84
85 start &= (PAGE_MASK << 1);
86 end += ((PAGE_SIZE << 1) - 1);
87 end &= (PAGE_MASK << 1);
88 while (start < end) {
89 int idx;
90
91 write_c0_entryhi(start | newpid);
92 start += (PAGE_SIZE << 1);
93 mtc0_tlbw_hazard();
94 tlb_probe();
95 BARRIER;
96 idx = read_c0_index();
97 write_c0_entrylo0(0);
98 write_c0_entrylo1(0);
99 if (idx < 0)
100 continue;
101 /* Make sure all entries differ. */
102 write_c0_entryhi(CKSEG0 +
103 (idx << (PAGE_SHIFT + 1)));
104 mtc0_tlbw_hazard();
105 tlb_write_indexed();
106 }
107 tlbw_use_hazard();
108 write_c0_entryhi(oldpid);
109 } else {
110 drop_mmu_context(mm, cpu);
111 }
112 local_irq_restore(flags);
113 }
114}
115
116void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
117{
118 unsigned long flags;
119 int size;
120
121 local_irq_save(flags);
122 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
123 size = (size + 1) >> 1;
124 if (size <= current_cpu_data.tlbsize / 2) {
125 int pid = read_c0_entryhi();
126
127 start &= (PAGE_MASK << 1);
128 end += ((PAGE_SIZE << 1) - 1);
129 end &= (PAGE_MASK << 1);
130
131 while (start < end) {
132 int idx;
133
134 write_c0_entryhi(start);
135 start += (PAGE_SIZE << 1);
136 mtc0_tlbw_hazard();
137 tlb_probe();
138 BARRIER;
139 idx = read_c0_index();
140 write_c0_entrylo0(0);
141 write_c0_entrylo1(0);
142 if (idx < 0)
143 continue;
144 /* Make sure all entries differ. */
145 write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
146 mtc0_tlbw_hazard();
147 tlb_write_indexed();
148 }
149 tlbw_use_hazard();
150 write_c0_entryhi(pid);
151 } else {
152 local_flush_tlb_all();
153 }
154 local_irq_restore(flags);
155}
156
157void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
158{
159 int cpu = smp_processor_id();
160
161 if (cpu_context(cpu, vma->vm_mm) != 0) {
162 unsigned long flags;
163 int oldpid, newpid, idx;
164
165 newpid = cpu_asid(cpu, vma->vm_mm);
166 page &= (PAGE_MASK << 1);
167 local_irq_save(flags);
168 oldpid = read_c0_entryhi();
169 write_c0_entryhi(page | newpid);
170 mtc0_tlbw_hazard();
171 tlb_probe();
172 BARRIER;
173 idx = read_c0_index();
174 write_c0_entrylo0(0);
175 write_c0_entrylo1(0);
176 if (idx < 0)
177 goto finish;
178 /* Make sure all entries differ. */
179 write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
180 mtc0_tlbw_hazard();
181 tlb_write_indexed();
182 tlbw_use_hazard();
183
184 finish:
185 write_c0_entryhi(oldpid);
186 local_irq_restore(flags);
187 }
188}
189
190/*
191 * This one is only used for pages with the global bit set so we don't care
192 * much about the ASID.
193 */
194void local_flush_tlb_one(unsigned long page)
195{
196 unsigned long flags;
197 int oldpid, idx;
198
199 local_irq_save(flags);
200 page &= (PAGE_MASK << 1);
201 oldpid = read_c0_entryhi();
202 write_c0_entryhi(page);
203 mtc0_tlbw_hazard();
204 tlb_probe();
205 BARRIER;
206 idx = read_c0_index();
207 write_c0_entrylo0(0);
208 write_c0_entrylo1(0);
209 if (idx >= 0) {
210 /* Make sure all entries differ. */
211 write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
212 mtc0_tlbw_hazard();
213 tlb_write_indexed();
214 tlbw_use_hazard();
215 }
216 write_c0_entryhi(oldpid);
217
218 local_irq_restore(flags);
219}
220
221/*
222 * We will need multiple versions of update_mmu_cache(), one that just
223 * updates the TLB with the new pte(s), and another which also checks
224 * for the R4k "end of page" hardware bug and does the needy.
225 */
226void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
227{
228 unsigned long flags;
229 pgd_t *pgdp;
230 pmd_t *pmdp;
231 pte_t *ptep;
232 int idx, pid;
233
234 /*
235 * Handle debugger faulting in for debugee.
236 */
237 if (current->active_mm != vma->vm_mm)
238 return;
239
240 pid = read_c0_entryhi() & ASID_MASK;
241
242 local_irq_save(flags);
243 address &= (PAGE_MASK << 1);
244 write_c0_entryhi(address | pid);
245 pgdp = pgd_offset(vma->vm_mm, address);
246 mtc0_tlbw_hazard();
247 tlb_probe();
248 BARRIER;
249 pmdp = pmd_offset(pgdp, address);
250 idx = read_c0_index();
251 ptep = pte_offset_map(pmdp, address);
252
253 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
254 write_c0_entrylo0(ptep->pte_high);
255 ptep++;
256 write_c0_entrylo1(ptep->pte_high);
257#else
258 write_c0_entrylo0(pte_val(*ptep++) >> 6);
259 write_c0_entrylo1(pte_val(*ptep) >> 6);
260#endif
261 write_c0_entryhi(address | pid);
262 mtc0_tlbw_hazard();
263 if (idx < 0)
264 tlb_write_random();
265 else
266 tlb_write_indexed();
267 tlbw_use_hazard();
268 write_c0_entryhi(pid);
269 local_irq_restore(flags);
270}
271
272#if 0
273static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma,
274 unsigned long address, pte_t pte)
275{
276 unsigned long flags;
277 unsigned int asid;
278 pgd_t *pgdp;
279 pmd_t *pmdp;
280 pte_t *ptep;
281 int idx;
282
283 local_irq_save(flags);
284 address &= (PAGE_MASK << 1);
285 asid = read_c0_entryhi() & ASID_MASK;
286 write_c0_entryhi(address | asid);
287 pgdp = pgd_offset(vma->vm_mm, address);
288 mtc0_tlbw_hazard();
289 tlb_probe();
290 BARRIER;
291 pmdp = pmd_offset(pgdp, address);
292 idx = read_c0_index();
293 ptep = pte_offset_map(pmdp, address);
294 write_c0_entrylo0(pte_val(*ptep++) >> 6);
295 write_c0_entrylo1(pte_val(*ptep) >> 6);
296 mtc0_tlbw_hazard();
297 if (idx < 0)
298 tlb_write_random();
299 else
300 tlb_write_indexed();
301 tlbw_use_hazard();
302 local_irq_restore(flags);
303}
304#endif
305
306void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
307 unsigned long entryhi, unsigned long pagemask)
308{
309 unsigned long flags;
310 unsigned long wired;
311 unsigned long old_pagemask;
312 unsigned long old_ctx;
313
314 local_irq_save(flags);
315 /* Save old context and create impossible VPN2 value */
316 old_ctx = read_c0_entryhi();
317 old_pagemask = read_c0_pagemask();
318 wired = read_c0_wired();
319 write_c0_wired(wired + 1);
320 write_c0_index(wired);
321 BARRIER;
322 write_c0_pagemask(pagemask);
323 write_c0_entryhi(entryhi);
324 write_c0_entrylo0(entrylo0);
325 write_c0_entrylo1(entrylo1);
326 mtc0_tlbw_hazard();
327 tlb_write_indexed();
328 tlbw_use_hazard();
329
330 write_c0_entryhi(old_ctx);
331 BARRIER;
332 write_c0_pagemask(old_pagemask);
333 local_flush_tlb_all();
334 local_irq_restore(flags);
335}
336
337/*
338 * Used for loading TLB entries before trap_init() has started, when we
339 * don't actually want to add a wired entry which remains throughout the
340 * lifetime of the system
341 */
342
343static int temp_tlb_entry __initdata;
344
345__init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
346 unsigned long entryhi, unsigned long pagemask)
347{
348 int ret = 0;
349 unsigned long flags;
350 unsigned long wired;
351 unsigned long old_pagemask;
352 unsigned long old_ctx;
353
354 local_irq_save(flags);
355 /* Save old context and create impossible VPN2 value */
356 old_ctx = read_c0_entryhi();
357 old_pagemask = read_c0_pagemask();
358 wired = read_c0_wired();
359 if (--temp_tlb_entry < wired) {
360 printk(KERN_WARNING "No TLB space left for add_temporary_entry\n");
361 ret = -ENOSPC;
362 goto out;
363 }
364
365 write_c0_index(temp_tlb_entry);
366 write_c0_pagemask(pagemask);
367 write_c0_entryhi(entryhi);
368 write_c0_entrylo0(entrylo0);
369 write_c0_entrylo1(entrylo1);
370 mtc0_tlbw_hazard();
371 tlb_write_indexed();
372 tlbw_use_hazard();
373
374 write_c0_entryhi(old_ctx);
375 write_c0_pagemask(old_pagemask);
376out:
377 local_irq_restore(flags);
378 return ret;
379}
380
381static void __init probe_tlb(unsigned long config)
382{
383 struct cpuinfo_mips *c = &current_cpu_data;
384 unsigned int reg;
385
386 /*
387 * If this isn't a MIPS32 / MIPS64 compliant CPU. Config 1 register
388 * is not supported, we assume R4k style. Cpu probing already figured
389 * out the number of tlb entries.
390 */
391 if ((c->processor_id & 0xff0000) == PRID_COMP_LEGACY)
392 return;
393
394 reg = read_c0_config1();
395 if (!((config >> 7) & 3))
396 panic("No TLB present");
397
398 c->tlbsize = ((reg >> 25) & 0x3f) + 1;
399}
400
401void __init tlb_init(void)
402{
403 unsigned int config = read_c0_config();
404
405 /*
406 * You should never change this register:
407 * - On R4600 1.7 the tlbp never hits for pages smaller than
408 * the value in the c0_pagemask register.
409 * - The entire mm handling assumes the c0_pagemask register to
410 * be set for 4kb pages.
411 */
412 probe_tlb(config);
413 write_c0_pagemask(PM_DEFAULT_MASK);
414 write_c0_wired(0);
415 temp_tlb_entry = current_cpu_data.tlbsize - 1;
416 local_flush_tlb_all();
417
418 build_tlb_refill_handler();
419}
diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c
new file mode 100644
index 000000000000..1bfb09198ce3
--- /dev/null
+++ b/arch/mips/mm/tlb-r8k.c
@@ -0,0 +1,250 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
8 * Carsten Langgaard, carstenl@mips.com
9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved.
10 */
11#include <linux/config.h>
12#include <linux/init.h>
13#include <linux/sched.h>
14#include <linux/mm.h>
15
16#include <asm/cpu.h>
17#include <asm/bootinfo.h>
18#include <asm/mmu_context.h>
19#include <asm/pgtable.h>
20#include <asm/system.h>
21
22extern void build_tlb_refill_handler(void);
23
24#define TFP_TLB_SIZE 384
25#define TFP_TLB_SET_SHIFT 7
26
27/* CP0 hazard avoidance. */
28#define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
29 "nop; nop; nop; nop; nop; nop;\n\t" \
30 ".set reorder\n\t")
31
32void local_flush_tlb_all(void)
33{
34 unsigned long flags;
35 unsigned long old_ctx;
36 int entry;
37
38 local_irq_save(flags);
39 /* Save old context and create impossible VPN2 value */
40 old_ctx = read_c0_entryhi();
41 write_c0_entrylo(0);
42
43 for (entry = 0; entry < TFP_TLB_SIZE; entry++) {
44 write_c0_tlbset(entry >> TFP_TLB_SET_SHIFT);
45 write_c0_vaddr(entry << PAGE_SHIFT);
46 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
47 mtc0_tlbw_hazard();
48 tlb_write();
49 }
50 tlbw_use_hazard();
51 write_c0_entryhi(old_ctx);
52 local_irq_restore(flags);
53}
54
55void local_flush_tlb_mm(struct mm_struct *mm)
56{
57 int cpu = smp_processor_id();
58
59 if (cpu_context(cpu, mm) != 0)
60 drop_mmu_context(mm,cpu);
61}
62
63void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
64 unsigned long end)
65{
66 struct mm_struct *mm = vma->vm_mm;
67 int cpu = smp_processor_id();
68 unsigned long flags;
69 int oldpid, newpid, size;
70
71 if (!cpu_context(cpu, mm))
72 return;
73
74 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
75 size = (size + 1) >> 1;
76
77 local_irq_save(flags);
78
79 if (size > TFP_TLB_SIZE / 2) {
80 drop_mmu_context(mm, cpu);
81 goto out_restore;
82 }
83
84 oldpid = read_c0_entryhi();
85 newpid = cpu_asid(cpu, mm);
86
87 write_c0_entrylo(0);
88
89 start &= PAGE_MASK;
90 end += (PAGE_SIZE - 1);
91 end &= PAGE_MASK;
92 while (start < end) {
93 signed long idx;
94
95 write_c0_vaddr(start);
96 write_c0_entryhi(start);
97 start += PAGE_SIZE;
98 tlb_probe();
99 idx = read_c0_tlbset();
100 if (idx < 0)
101 continue;
102
103 write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
104 tlb_write();
105 }
106 write_c0_entryhi(oldpid);
107
108out_restore:
109 local_irq_restore(flags);
110}
111
112/* Usable for KV1 addresses only! */
113void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
114{
115 unsigned long flags;
116 int size;
117
118 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
119 size = (size + 1) >> 1;
120
121 if (size > TFP_TLB_SIZE / 2) {
122 local_flush_tlb_all();
123 return;
124 }
125
126 local_irq_save(flags);
127
128 write_c0_entrylo(0);
129
130 start &= PAGE_MASK;
131 end += (PAGE_SIZE - 1);
132 end &= PAGE_MASK;
133 while (start < end) {
134 signed long idx;
135
136 write_c0_vaddr(start);
137 write_c0_entryhi(start);
138 start += PAGE_SIZE;
139 tlb_probe();
140 idx = read_c0_tlbset();
141 if (idx < 0)
142 continue;
143
144 write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
145 tlb_write();
146 }
147
148 local_irq_restore(flags);
149}
150
151void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
152{
153 int cpu = smp_processor_id();
154 unsigned long flags;
155 int oldpid, newpid;
156 signed long idx;
157
158 if (!cpu_context(cpu, vma->vm_mm))
159 return;
160
161 newpid = cpu_asid(cpu, vma->vm_mm);
162 page &= PAGE_MASK;
163 local_irq_save(flags);
164 oldpid = read_c0_entryhi();
165 write_c0_vaddr(page);
166 write_c0_entryhi(newpid);
167 tlb_probe();
168 idx = read_c0_tlbset();
169 if (idx < 0)
170 goto finish;
171
172 write_c0_entrylo(0);
173 write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
174 tlb_write();
175
176finish:
177 write_c0_entryhi(oldpid);
178 local_irq_restore(flags);
179}
180
181/*
182 * We will need multiple versions of update_mmu_cache(), one that just
183 * updates the TLB with the new pte(s), and another which also checks
184 * for the R4k "end of page" hardware bug and does the needy.
185 */
186void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
187{
188 unsigned long flags;
189 pgd_t *pgdp;
190 pmd_t *pmdp;
191 pte_t *ptep;
192 int pid;
193
194 /*
195 * Handle debugger faulting in for debugee.
196 */
197 if (current->active_mm != vma->vm_mm)
198 return;
199
200 pid = read_c0_entryhi() & ASID_MASK;
201
202 local_irq_save(flags);
203 address &= PAGE_MASK;
204 write_c0_vaddr(address);
205 write_c0_entryhi(pid);
206 pgdp = pgd_offset(vma->vm_mm, address);
207 pmdp = pmd_offset(pgdp, address);
208 ptep = pte_offset_map(pmdp, address);
209 tlb_probe();
210
211 write_c0_entrylo(pte_val(*ptep++) >> 6);
212 tlb_write();
213
214 write_c0_entryhi(pid);
215 local_irq_restore(flags);
216}
217
218static void __init probe_tlb(unsigned long config)
219{
220 struct cpuinfo_mips *c = &current_cpu_data;
221
222 c->tlbsize = 3 * 128; /* 3 sets each 128 entries */
223}
224
225void __init tlb_init(void)
226{
227 unsigned int config = read_c0_config();
228 unsigned long status;
229
230 probe_tlb(config);
231
232 status = read_c0_status();
233 status &= ~(ST0_UPS | ST0_KPS);
234#ifdef CONFIG_PAGE_SIZE_4KB
235 status |= (TFP_PAGESIZE_4K << 32) | (TFP_PAGESIZE_4K << 36);
236#elif defined(CONFIG_PAGE_SIZE_8KB)
237 status |= (TFP_PAGESIZE_8K << 32) | (TFP_PAGESIZE_8K << 36);
238#elif defined(CONFIG_PAGE_SIZE_16KB)
239 status |= (TFP_PAGESIZE_16K << 32) | (TFP_PAGESIZE_16K << 36);
240#elif defined(CONFIG_PAGE_SIZE_64KB)
241 status |= (TFP_PAGESIZE_64K << 32) | (TFP_PAGESIZE_64K << 36);
242#endif
243 write_c0_status(status);
244
245 write_c0_wired(0);
246
247 local_flush_tlb_all();
248
249 build_tlb_refill_handler();
250}
diff --git a/arch/mips/mm/tlb-sb1.c b/arch/mips/mm/tlb-sb1.c
new file mode 100644
index 000000000000..6256cafcf3a2
--- /dev/null
+++ b/arch/mips/mm/tlb-sb1.c
@@ -0,0 +1,376 @@
1/*
2 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
3 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org)
4 * Copyright (C) 2000, 2001, 2002, 2003 Broadcom Corporation
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 */
20#include <linux/init.h>
21#include <asm/mmu_context.h>
22#include <asm/bootinfo.h>
23#include <asm/cpu.h>
24
25extern void build_tlb_refill_handler(void);
26
27#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
28
29/* Dump the current entry* and pagemask registers */
30static inline void dump_cur_tlb_regs(void)
31{
32 unsigned int entryhihi, entryhilo, entrylo0hi, entrylo0lo, entrylo1hi;
33 unsigned int entrylo1lo, pagemask;
34
35 __asm__ __volatile__ (
36 ".set push \n"
37 ".set noreorder \n"
38 ".set mips64 \n"
39 ".set noat \n"
40 " tlbr \n"
41 " dmfc0 $1, $10 \n"
42 " dsrl32 %0, $1, 0 \n"
43 " sll %1, $1, 0 \n"
44 " dmfc0 $1, $2 \n"
45 " dsrl32 %2, $1, 0 \n"
46 " sll %3, $1, 0 \n"
47 " dmfc0 $1, $3 \n"
48 " dsrl32 %4, $1, 0 \n"
49 " sll %5, $1, 0 \n"
50 " mfc0 %6, $5 \n"
51 ".set pop \n"
52 : "=r" (entryhihi), "=r" (entryhilo),
53 "=r" (entrylo0hi), "=r" (entrylo0lo),
54 "=r" (entrylo1hi), "=r" (entrylo1lo),
55 "=r" (pagemask));
56
57 printk("%08X%08X %08X%08X %08X%08X %08X",
58 entryhihi, entryhilo,
59 entrylo0hi, entrylo0lo,
60 entrylo1hi, entrylo1lo,
61 pagemask);
62}
63
64void sb1_dump_tlb(void)
65{
66 unsigned long old_ctx;
67 unsigned long flags;
68 int entry;
69 local_irq_save(flags);
70 old_ctx = read_c0_entryhi();
71 printk("Current TLB registers state:\n"
72 " EntryHi EntryLo0 EntryLo1 PageMask Index\n"
73 "--------------------------------------------------------------------\n");
74 dump_cur_tlb_regs();
75 printk(" %08X\n", read_c0_index());
76 printk("\n\nFull TLB Dump:\n"
77 "Idx EntryHi EntryLo0 EntryLo1 PageMask\n"
78 "--------------------------------------------------------------\n");
79 for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
80 write_c0_index(entry);
81 printk("\n%02i ", entry);
82 dump_cur_tlb_regs();
83 }
84 printk("\n");
85 write_c0_entryhi(old_ctx);
86 local_irq_restore(flags);
87}
88
89void local_flush_tlb_all(void)
90{
91 unsigned long flags;
92 unsigned long old_ctx;
93 int entry;
94
95 local_irq_save(flags);
96 /* Save old context and create impossible VPN2 value */
97 old_ctx = read_c0_entryhi() & ASID_MASK;
98 write_c0_entrylo0(0);
99 write_c0_entrylo1(0);
100
101 entry = read_c0_wired();
102 while (entry < current_cpu_data.tlbsize) {
103 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
104 write_c0_index(entry);
105 tlb_write_indexed();
106 entry++;
107 }
108 write_c0_entryhi(old_ctx);
109 local_irq_restore(flags);
110}
111
112
113/*
114 * Use a bogus region of memory (starting at 0) to sanitize the TLB's.
115 * Use increments of the maximum page size (16MB), and check for duplicate
116 * entries before doing a given write. Then, when we're safe from collisions
117 * with the firmware, go back and give all the entries invalid addresses with
118 * the normal flush routine. Wired entries will be killed as well!
119 */
120static void __init sb1_sanitize_tlb(void)
121{
122 int entry;
123 long addr = 0;
124
125 long inc = 1<<24; /* 16MB */
126 /* Save old context and create impossible VPN2 value */
127 write_c0_entrylo0(0);
128 write_c0_entrylo1(0);
129 for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
130 do {
131 addr += inc;
132 write_c0_entryhi(addr);
133 tlb_probe();
134 } while ((int)(read_c0_index()) >= 0);
135 write_c0_index(entry);
136 tlb_write_indexed();
137 }
138 /* Now that we know we're safe from collisions, we can safely flush
139 the TLB with the "normal" routine. */
140 local_flush_tlb_all();
141}
142
143void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
144 unsigned long end)
145{
146 struct mm_struct *mm = vma->vm_mm;
147 unsigned long flags;
148 int cpu;
149
150 local_irq_save(flags);
151 cpu = smp_processor_id();
152 if (cpu_context(cpu, mm) != 0) {
153 int size;
154 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
155 size = (size + 1) >> 1;
156 if (size <= (current_cpu_data.tlbsize/2)) {
157 int oldpid = read_c0_entryhi() & ASID_MASK;
158 int newpid = cpu_asid(cpu, mm);
159
160 start &= (PAGE_MASK << 1);
161 end += ((PAGE_SIZE << 1) - 1);
162 end &= (PAGE_MASK << 1);
163 while (start < end) {
164 int idx;
165
166 write_c0_entryhi(start | newpid);
167 start += (PAGE_SIZE << 1);
168 tlb_probe();
169 idx = read_c0_index();
170 write_c0_entrylo0(0);
171 write_c0_entrylo1(0);
172 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
173 if (idx < 0)
174 continue;
175 tlb_write_indexed();
176 }
177 write_c0_entryhi(oldpid);
178 } else {
179 drop_mmu_context(mm, cpu);
180 }
181 }
182 local_irq_restore(flags);
183}
184
185void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
186{
187 unsigned long flags;
188 int size;
189
190 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
191 size = (size + 1) >> 1;
192
193 local_irq_save(flags);
194 if (size <= (current_cpu_data.tlbsize/2)) {
195 int pid = read_c0_entryhi();
196
197 start &= (PAGE_MASK << 1);
198 end += ((PAGE_SIZE << 1) - 1);
199 end &= (PAGE_MASK << 1);
200
201 while (start < end) {
202 int idx;
203
204 write_c0_entryhi(start);
205 start += (PAGE_SIZE << 1);
206 tlb_probe();
207 idx = read_c0_index();
208 write_c0_entrylo0(0);
209 write_c0_entrylo1(0);
210 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
211 if (idx < 0)
212 continue;
213 tlb_write_indexed();
214 }
215 write_c0_entryhi(pid);
216 } else {
217 local_flush_tlb_all();
218 }
219 local_irq_restore(flags);
220}
221
222void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
223{
224 unsigned long flags;
225 int cpu = smp_processor_id();
226
227 local_irq_save(flags);
228 if (cpu_context(cpu, vma->vm_mm) != 0) {
229 int oldpid, newpid, idx;
230 newpid = cpu_asid(cpu, vma->vm_mm);
231 page &= (PAGE_MASK << 1);
232 oldpid = read_c0_entryhi() & ASID_MASK;
233 write_c0_entryhi(page | newpid);
234 tlb_probe();
235 idx = read_c0_index();
236 write_c0_entrylo0(0);
237 write_c0_entrylo1(0);
238 if (idx < 0)
239 goto finish;
240 /* Make sure all entries differ. */
241 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
242 tlb_write_indexed();
243 finish:
244 write_c0_entryhi(oldpid);
245 }
246 local_irq_restore(flags);
247}
248
249/*
250 * Remove one kernel space TLB entry. This entry is assumed to be marked
251 * global so we don't do the ASID thing.
252 */
253void local_flush_tlb_one(unsigned long page)
254{
255 unsigned long flags;
256 int oldpid, idx;
257
258 page &= (PAGE_MASK << 1);
259 oldpid = read_c0_entryhi() & ASID_MASK;
260
261 local_irq_save(flags);
262 write_c0_entryhi(page);
263 tlb_probe();
264 idx = read_c0_index();
265 if (idx >= 0) {
266 /* Make sure all entries differ. */
267 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
268 write_c0_entrylo0(0);
269 write_c0_entrylo1(0);
270 tlb_write_indexed();
271 }
272
273 write_c0_entryhi(oldpid);
274 local_irq_restore(flags);
275}
276
277/* All entries common to a mm share an asid. To effectively flush
278 these entries, we just bump the asid. */
279void local_flush_tlb_mm(struct mm_struct *mm)
280{
281 int cpu;
282
283 preempt_disable();
284
285 cpu = smp_processor_id();
286
287 if (cpu_context(cpu, mm) != 0) {
288 drop_mmu_context(mm, cpu);
289 }
290
291 preempt_enable();
292}
293
294/* Stolen from mips32 routines */
295
296void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
297{
298 unsigned long flags;
299 pgd_t *pgdp;
300 pmd_t *pmdp;
301 pte_t *ptep;
302 int idx, pid;
303
304 /*
305 * Handle debugger faulting in for debugee.
306 */
307 if (current->active_mm != vma->vm_mm)
308 return;
309
310 local_irq_save(flags);
311
312 pid = read_c0_entryhi() & ASID_MASK;
313 address &= (PAGE_MASK << 1);
314 write_c0_entryhi(address | (pid));
315 pgdp = pgd_offset(vma->vm_mm, address);
316 tlb_probe();
317 pmdp = pmd_offset(pgdp, address);
318 idx = read_c0_index();
319 ptep = pte_offset_map(pmdp, address);
320 write_c0_entrylo0(pte_val(*ptep++) >> 6);
321 write_c0_entrylo1(pte_val(*ptep) >> 6);
322 if (idx < 0) {
323 tlb_write_random();
324 } else {
325 tlb_write_indexed();
326 }
327 local_irq_restore(flags);
328}
329
330void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
331 unsigned long entryhi, unsigned long pagemask)
332{
333 unsigned long flags;
334 unsigned long wired;
335 unsigned long old_pagemask;
336 unsigned long old_ctx;
337
338 local_irq_save(flags);
339 old_ctx = read_c0_entryhi() & 0xff;
340 old_pagemask = read_c0_pagemask();
341 wired = read_c0_wired();
342 write_c0_wired(wired + 1);
343 write_c0_index(wired);
344
345 write_c0_pagemask(pagemask);
346 write_c0_entryhi(entryhi);
347 write_c0_entrylo0(entrylo0);
348 write_c0_entrylo1(entrylo1);
349 tlb_write_indexed();
350
351 write_c0_entryhi(old_ctx);
352 write_c0_pagemask(old_pagemask);
353
354 local_flush_tlb_all();
355 local_irq_restore(flags);
356}
357
358/*
359 * This is called from loadmmu.c. We have to set up all the
360 * memory management function pointers, as well as initialize
361 * the caches and tlbs
362 */
363void tlb_init(void)
364{
365 write_c0_pagemask(PM_DEFAULT_MASK);
366 write_c0_wired(0);
367
368 /*
369 * We don't know what state the firmware left the TLB's in, so this is
370 * the ultra-conservative way to flush the TLB's and avoid machine
371 * check exceptions due to duplicate TLB entries
372 */
373 sb1_sanitize_tlb();
374
375 build_tlb_refill_handler();
376}
diff --git a/arch/mips/mm/tlbex-fault.S b/arch/mips/mm/tlbex-fault.S
new file mode 100644
index 000000000000..9e7f4175b493
--- /dev/null
+++ b/arch/mips/mm/tlbex-fault.S
@@ -0,0 +1,28 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999 Ralf Baechle
7 * Copyright (C) 1999 Silicon Graphics, Inc.
8 */
9#include <asm/mipsregs.h>
10#include <asm/page.h>
11#include <asm/regdef.h>
12#include <asm/stackframe.h>
13
14 .macro tlb_do_page_fault, write
15 NESTED(tlb_do_page_fault_\write, PT_SIZE, sp)
16 SAVE_ALL
17 MFC0 a2, CP0_BADVADDR
18 KMODE
19 move a0, sp
20 REG_S a2, PT_BVADDR(sp)
21 li a1, \write
22 jal do_page_fault
23 j ret_from_exception
24 END(tlb_do_page_fault_\write)
25 .endm
26
27 tlb_do_page_fault 0
28 tlb_do_page_fault 1
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
new file mode 100644
index 000000000000..87e229f4d3d5
--- /dev/null
+++ b/arch/mips/mm/tlbex.c
@@ -0,0 +1,1815 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Synthesize TLB refill handlers at runtime.
7 *
8 * Copyright (C) 2004,2005 by Thiemo Seufer
9 */
10
11#include <stdarg.h>
12
13#include <linux/config.h>
14#include <linux/mm.h>
15#include <linux/kernel.h>
16#include <linux/types.h>
17#include <linux/string.h>
18#include <linux/init.h>
19
20#include <asm/pgtable.h>
21#include <asm/cacheflush.h>
22#include <asm/mmu_context.h>
23#include <asm/inst.h>
24#include <asm/elf.h>
25#include <asm/smp.h>
26#include <asm/war.h>
27
28/* #define DEBUG_TLB */
29
30static __init int __attribute__((unused)) r45k_bvahwbug(void)
31{
32 /* XXX: We should probe for the presence of this bug, but we don't. */
33 return 0;
34}
35
36static __init int __attribute__((unused)) r4k_250MHZhwbug(void)
37{
38 /* XXX: We should probe for the presence of this bug, but we don't. */
39 return 0;
40}
41
42static __init int __attribute__((unused)) bcm1250_m3_war(void)
43{
44 return BCM1250_M3_WAR;
45}
46
47static __init int __attribute__((unused)) r10000_llsc_war(void)
48{
49 return R10000_LLSC_WAR;
50}
51
52/*
53 * A little micro-assembler, intended for TLB refill handler
54 * synthesizing. It is intentionally kept simple, does only support
55 * a subset of instructions, and does not try to hide pipeline effects
56 * like branch delay slots.
57 */
58
59enum fields
60{
61 RS = 0x001,
62 RT = 0x002,
63 RD = 0x004,
64 RE = 0x008,
65 SIMM = 0x010,
66 UIMM = 0x020,
67 BIMM = 0x040,
68 JIMM = 0x080,
69 FUNC = 0x100,
70};
71
72#define OP_MASK 0x2f
73#define OP_SH 26
74#define RS_MASK 0x1f
75#define RS_SH 21
76#define RT_MASK 0x1f
77#define RT_SH 16
78#define RD_MASK 0x1f
79#define RD_SH 11
80#define RE_MASK 0x1f
81#define RE_SH 6
82#define IMM_MASK 0xffff
83#define IMM_SH 0
84#define JIMM_MASK 0x3ffffff
85#define JIMM_SH 0
86#define FUNC_MASK 0x2f
87#define FUNC_SH 0
88
89enum opcode {
90 insn_invalid,
91 insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
92 insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
93 insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
94 insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
95 insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
96 insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
97 insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
98 insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi,
99 insn_tlbwr, insn_xor, insn_xori
100};
101
102struct insn {
103 enum opcode opcode;
104 u32 match;
105 enum fields fields;
106};
107
108/* This macro sets the non-variable bits of an instruction. */
109#define M(a, b, c, d, e, f) \
110 ((a) << OP_SH \
111 | (b) << RS_SH \
112 | (c) << RT_SH \
113 | (d) << RD_SH \
114 | (e) << RE_SH \
115 | (f) << FUNC_SH)
116
117static __initdata struct insn insn_table[] = {
118 { insn_addiu, M(addiu_op,0,0,0,0,0), RS | RT | SIMM },
119 { insn_addu, M(spec_op,0,0,0,0,addu_op), RS | RT | RD },
120 { insn_and, M(spec_op,0,0,0,0,and_op), RS | RT | RD },
121 { insn_andi, M(andi_op,0,0,0,0,0), RS | RT | UIMM },
122 { insn_beq, M(beq_op,0,0,0,0,0), RS | RT | BIMM },
123 { insn_beql, M(beql_op,0,0,0,0,0), RS | RT | BIMM },
124 { insn_bgez, M(bcond_op,0,bgez_op,0,0,0), RS | BIMM },
125 { insn_bgezl, M(bcond_op,0,bgezl_op,0,0,0), RS | BIMM },
126 { insn_bltz, M(bcond_op,0,bltz_op,0,0,0), RS | BIMM },
127 { insn_bltzl, M(bcond_op,0,bltzl_op,0,0,0), RS | BIMM },
128 { insn_bne, M(bne_op,0,0,0,0,0), RS | RT | BIMM },
129 { insn_daddiu, M(daddiu_op,0,0,0,0,0), RS | RT | SIMM },
130 { insn_daddu, M(spec_op,0,0,0,0,daddu_op), RS | RT | RD },
131 { insn_dmfc0, M(cop0_op,dmfc_op,0,0,0,0), RT | RD },
132 { insn_dmtc0, M(cop0_op,dmtc_op,0,0,0,0), RT | RD },
133 { insn_dsll, M(spec_op,0,0,0,0,dsll_op), RT | RD | RE },
134 { insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE },
135 { insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE },
136 { insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE },
137 { insn_dsrl32, M(spec_op,0,0,0,0,dsrl32_op), RT | RD | RE },
138 { insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD },
139 { insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 },
140 { insn_j, M(j_op,0,0,0,0,0), JIMM },
141 { insn_jal, M(jal_op,0,0,0,0,0), JIMM },
142 { insn_jr, M(spec_op,0,0,0,0,jr_op), RS },
143 { insn_ld, M(ld_op,0,0,0,0,0), RS | RT | SIMM },
144 { insn_ll, M(ll_op,0,0,0,0,0), RS | RT | SIMM },
145 { insn_lld, M(lld_op,0,0,0,0,0), RS | RT | SIMM },
146 { insn_lui, M(lui_op,0,0,0,0,0), RT | SIMM },
147 { insn_lw, M(lw_op,0,0,0,0,0), RS | RT | SIMM },
148 { insn_mfc0, M(cop0_op,mfc_op,0,0,0,0), RT | RD },
149 { insn_mtc0, M(cop0_op,mtc_op,0,0,0,0), RT | RD },
150 { insn_ori, M(ori_op,0,0,0,0,0), RS | RT | UIMM },
151 { insn_rfe, M(cop0_op,cop_op,0,0,0,rfe_op), 0 },
152 { insn_sc, M(sc_op,0,0,0,0,0), RS | RT | SIMM },
153 { insn_scd, M(scd_op,0,0,0,0,0), RS | RT | SIMM },
154 { insn_sd, M(sd_op,0,0,0,0,0), RS | RT | SIMM },
155 { insn_sll, M(spec_op,0,0,0,0,sll_op), RT | RD | RE },
156 { insn_sra, M(spec_op,0,0,0,0,sra_op), RT | RD | RE },
157 { insn_srl, M(spec_op,0,0,0,0,srl_op), RT | RD | RE },
158 { insn_subu, M(spec_op,0,0,0,0,subu_op), RS | RT | RD },
159 { insn_sw, M(sw_op,0,0,0,0,0), RS | RT | SIMM },
160 { insn_tlbp, M(cop0_op,cop_op,0,0,0,tlbp_op), 0 },
161 { insn_tlbwi, M(cop0_op,cop_op,0,0,0,tlbwi_op), 0 },
162 { insn_tlbwr, M(cop0_op,cop_op,0,0,0,tlbwr_op), 0 },
163 { insn_xor, M(spec_op,0,0,0,0,xor_op), RS | RT | RD },
164 { insn_xori, M(xori_op,0,0,0,0,0), RS | RT | UIMM },
165 { insn_invalid, 0, 0 }
166};
167
168#undef M
169
170static __init u32 build_rs(u32 arg)
171{
172 if (arg & ~RS_MASK)
173 printk(KERN_WARNING "TLB synthesizer field overflow\n");
174
175 return (arg & RS_MASK) << RS_SH;
176}
177
178static __init u32 build_rt(u32 arg)
179{
180 if (arg & ~RT_MASK)
181 printk(KERN_WARNING "TLB synthesizer field overflow\n");
182
183 return (arg & RT_MASK) << RT_SH;
184}
185
186static __init u32 build_rd(u32 arg)
187{
188 if (arg & ~RD_MASK)
189 printk(KERN_WARNING "TLB synthesizer field overflow\n");
190
191 return (arg & RD_MASK) << RD_SH;
192}
193
194static __init u32 build_re(u32 arg)
195{
196 if (arg & ~RE_MASK)
197 printk(KERN_WARNING "TLB synthesizer field overflow\n");
198
199 return (arg & RE_MASK) << RE_SH;
200}
201
202static __init u32 build_simm(s32 arg)
203{
204 if (arg > 0x7fff || arg < -0x8000)
205 printk(KERN_WARNING "TLB synthesizer field overflow\n");
206
207 return arg & 0xffff;
208}
209
210static __init u32 build_uimm(u32 arg)
211{
212 if (arg & ~IMM_MASK)
213 printk(KERN_WARNING "TLB synthesizer field overflow\n");
214
215 return arg & IMM_MASK;
216}
217
218static __init u32 build_bimm(s32 arg)
219{
220 if (arg > 0x1ffff || arg < -0x20000)
221 printk(KERN_WARNING "TLB synthesizer field overflow\n");
222
223 if (arg & 0x3)
224 printk(KERN_WARNING "Invalid TLB synthesizer branch target\n");
225
226 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
227}
228
229static __init u32 build_jimm(u32 arg)
230{
231 if (arg & ~((JIMM_MASK) << 2))
232 printk(KERN_WARNING "TLB synthesizer field overflow\n");
233
234 return (arg >> 2) & JIMM_MASK;
235}
236
237static __init u32 build_func(u32 arg)
238{
239 if (arg & ~FUNC_MASK)
240 printk(KERN_WARNING "TLB synthesizer field overflow\n");
241
242 return arg & FUNC_MASK;
243}
244
245/*
246 * The order of opcode arguments is implicitly left to right,
247 * starting with RS and ending with FUNC or IMM.
248 */
249static void __init build_insn(u32 **buf, enum opcode opc, ...)
250{
251 struct insn *ip = NULL;
252 unsigned int i;
253 va_list ap;
254 u32 op;
255
256 for (i = 0; insn_table[i].opcode != insn_invalid; i++)
257 if (insn_table[i].opcode == opc) {
258 ip = &insn_table[i];
259 break;
260 }
261
262 if (!ip)
263 panic("Unsupported TLB synthesizer instruction %d", opc);
264
265 op = ip->match;
266 va_start(ap, opc);
267 if (ip->fields & RS) op |= build_rs(va_arg(ap, u32));
268 if (ip->fields & RT) op |= build_rt(va_arg(ap, u32));
269 if (ip->fields & RD) op |= build_rd(va_arg(ap, u32));
270 if (ip->fields & RE) op |= build_re(va_arg(ap, u32));
271 if (ip->fields & SIMM) op |= build_simm(va_arg(ap, s32));
272 if (ip->fields & UIMM) op |= build_uimm(va_arg(ap, u32));
273 if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32));
274 if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32));
275 if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32));
276 va_end(ap);
277
278 **buf = op;
279 (*buf)++;
280}
281
282#define I_u1u2u3(op) \
283 static inline void i##op(u32 **buf, unsigned int a, \
284 unsigned int b, unsigned int c) \
285 { \
286 build_insn(buf, insn##op, a, b, c); \
287 }
288
289#define I_u2u1u3(op) \
290 static inline void i##op(u32 **buf, unsigned int a, \
291 unsigned int b, unsigned int c) \
292 { \
293 build_insn(buf, insn##op, b, a, c); \
294 }
295
296#define I_u3u1u2(op) \
297 static inline void i##op(u32 **buf, unsigned int a, \
298 unsigned int b, unsigned int c) \
299 { \
300 build_insn(buf, insn##op, b, c, a); \
301 }
302
303#define I_u1u2s3(op) \
304 static inline void i##op(u32 **buf, unsigned int a, \
305 unsigned int b, signed int c) \
306 { \
307 build_insn(buf, insn##op, a, b, c); \
308 }
309
310#define I_u2s3u1(op) \
311 static inline void i##op(u32 **buf, unsigned int a, \
312 signed int b, unsigned int c) \
313 { \
314 build_insn(buf, insn##op, c, a, b); \
315 }
316
317#define I_u2u1s3(op) \
318 static inline void i##op(u32 **buf, unsigned int a, \
319 unsigned int b, signed int c) \
320 { \
321 build_insn(buf, insn##op, b, a, c); \
322 }
323
324#define I_u1u2(op) \
325 static inline void i##op(u32 **buf, unsigned int a, \
326 unsigned int b) \
327 { \
328 build_insn(buf, insn##op, a, b); \
329 }
330
331#define I_u1s2(op) \
332 static inline void i##op(u32 **buf, unsigned int a, \
333 signed int b) \
334 { \
335 build_insn(buf, insn##op, a, b); \
336 }
337
338#define I_u1(op) \
339 static inline void i##op(u32 **buf, unsigned int a) \
340 { \
341 build_insn(buf, insn##op, a); \
342 }
343
344#define I_0(op) \
345 static inline void i##op(u32 **buf) \
346 { \
347 build_insn(buf, insn##op); \
348 }
349
350I_u2u1s3(_addiu);
351I_u3u1u2(_addu);
352I_u2u1u3(_andi);
353I_u3u1u2(_and);
354I_u1u2s3(_beq);
355I_u1u2s3(_beql);
356I_u1s2(_bgez);
357I_u1s2(_bgezl);
358I_u1s2(_bltz);
359I_u1s2(_bltzl);
360I_u1u2s3(_bne);
361I_u1u2(_dmfc0);
362I_u1u2(_dmtc0);
363I_u2u1s3(_daddiu);
364I_u3u1u2(_daddu);
365I_u2u1u3(_dsll);
366I_u2u1u3(_dsll32);
367I_u2u1u3(_dsra);
368I_u2u1u3(_dsrl);
369I_u2u1u3(_dsrl32);
370I_u3u1u2(_dsubu);
371I_0(_eret);
372I_u1(_j);
373I_u1(_jal);
374I_u1(_jr);
375I_u2s3u1(_ld);
376I_u2s3u1(_ll);
377I_u2s3u1(_lld);
378I_u1s2(_lui);
379I_u2s3u1(_lw);
380I_u1u2(_mfc0);
381I_u1u2(_mtc0);
382I_u2u1u3(_ori);
383I_0(_rfe);
384I_u2s3u1(_sc);
385I_u2s3u1(_scd);
386I_u2s3u1(_sd);
387I_u2u1u3(_sll);
388I_u2u1u3(_sra);
389I_u2u1u3(_srl);
390I_u3u1u2(_subu);
391I_u2s3u1(_sw);
392I_0(_tlbp);
393I_0(_tlbwi);
394I_0(_tlbwr);
395I_u3u1u2(_xor)
396I_u2u1u3(_xori);
397
398/*
399 * handling labels
400 */
401
402enum label_id {
403 label_invalid,
404 label_second_part,
405 label_leave,
406 label_vmalloc,
407 label_vmalloc_done,
408 label_tlbw_hazard,
409 label_split,
410 label_nopage_tlbl,
411 label_nopage_tlbs,
412 label_nopage_tlbm,
413 label_smp_pgtable_change,
414 label_r3000_write_probe_fail,
415 label_r3000_write_probe_ok
416};
417
418struct label {
419 u32 *addr;
420 enum label_id lab;
421};
422
423static __init void build_label(struct label **lab, u32 *addr,
424 enum label_id l)
425{
426 (*lab)->addr = addr;
427 (*lab)->lab = l;
428 (*lab)++;
429}
430
431#define L_LA(lb) \
432 static inline void l##lb(struct label **lab, u32 *addr) \
433 { \
434 build_label(lab, addr, label##lb); \
435 }
436
437L_LA(_second_part)
438L_LA(_leave)
439L_LA(_vmalloc)
440L_LA(_vmalloc_done)
441L_LA(_tlbw_hazard)
442L_LA(_split)
443L_LA(_nopage_tlbl)
444L_LA(_nopage_tlbs)
445L_LA(_nopage_tlbm)
446L_LA(_smp_pgtable_change)
447L_LA(_r3000_write_probe_fail)
448L_LA(_r3000_write_probe_ok)
449
450/* convenience macros for instructions */
451#ifdef CONFIG_MIPS64
452# define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off)
453# define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off)
454# define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh)
455# define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh)
456# define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh)
457# define i_MFC0(buf, rt, rd) i_dmfc0(buf, rt, rd)
458# define i_MTC0(buf, rt, rd) i_dmtc0(buf, rt, rd)
459# define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val)
460# define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd)
461# define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd)
462# define i_LL(buf, rs, rt, off) i_lld(buf, rs, rt, off)
463# define i_SC(buf, rs, rt, off) i_scd(buf, rs, rt, off)
464#else
465# define i_LW(buf, rs, rt, off) i_lw(buf, rs, rt, off)
466# define i_SW(buf, rs, rt, off) i_sw(buf, rs, rt, off)
467# define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh)
468# define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh)
469# define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh)
470# define i_MFC0(buf, rt, rd) i_mfc0(buf, rt, rd)
471# define i_MTC0(buf, rt, rd) i_mtc0(buf, rt, rd)
472# define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val)
473# define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd)
474# define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd)
475# define i_LL(buf, rs, rt, off) i_ll(buf, rs, rt, off)
476# define i_SC(buf, rs, rt, off) i_sc(buf, rs, rt, off)
477#endif
478
479#define i_b(buf, off) i_beq(buf, 0, 0, off)
480#define i_beqz(buf, rs, off) i_beq(buf, rs, 0, off)
481#define i_beqzl(buf, rs, off) i_beql(buf, rs, 0, off)
482#define i_bnez(buf, rs, off) i_bne(buf, rs, 0, off)
483#define i_bnezl(buf, rs, off) i_bnel(buf, rs, 0, off)
484#define i_move(buf, a, b) i_ADDU(buf, a, 0, b)
485#define i_nop(buf) i_sll(buf, 0, 0, 0)
486#define i_ssnop(buf) i_sll(buf, 0, 0, 1)
487#define i_ehb(buf) i_sll(buf, 0, 0, 3)
488
489#ifdef CONFIG_MIPS64
490static __init int __attribute__((unused)) in_compat_space_p(long addr)
491{
492 /* Is this address in 32bit compat space? */
493 return (((addr) & 0xffffffff00000000) == 0xffffffff00000000);
494}
495
496static __init int __attribute__((unused)) rel_highest(long val)
497{
498 return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
499}
500
501static __init int __attribute__((unused)) rel_higher(long val)
502{
503 return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
504}
505#endif
506
507static __init int rel_hi(long val)
508{
509 return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
510}
511
512static __init int rel_lo(long val)
513{
514 return ((val & 0xffff) ^ 0x8000) - 0x8000;
515}
516
517static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr)
518{
519#if CONFIG_MIPS64
520 if (!in_compat_space_p(addr)) {
521 i_lui(buf, rs, rel_highest(addr));
522 if (rel_higher(addr))
523 i_daddiu(buf, rs, rs, rel_higher(addr));
524 if (rel_hi(addr)) {
525 i_dsll(buf, rs, rs, 16);
526 i_daddiu(buf, rs, rs, rel_hi(addr));
527 i_dsll(buf, rs, rs, 16);
528 } else
529 i_dsll32(buf, rs, rs, 0);
530 } else
531#endif
532 i_lui(buf, rs, rel_hi(addr));
533}
534
535static __init void __attribute__((unused)) i_LA(u32 **buf, unsigned int rs,
536 long addr)
537{
538 i_LA_mostly(buf, rs, addr);
539 if (rel_lo(addr))
540 i_ADDIU(buf, rs, rs, rel_lo(addr));
541}
542
543/*
544 * handle relocations
545 */
546
547struct reloc {
548 u32 *addr;
549 unsigned int type;
550 enum label_id lab;
551};
552
553static __init void r_mips_pc16(struct reloc **rel, u32 *addr,
554 enum label_id l)
555{
556 (*rel)->addr = addr;
557 (*rel)->type = R_MIPS_PC16;
558 (*rel)->lab = l;
559 (*rel)++;
560}
561
562static inline void __resolve_relocs(struct reloc *rel, struct label *lab)
563{
564 long laddr = (long)lab->addr;
565 long raddr = (long)rel->addr;
566
567 switch (rel->type) {
568 case R_MIPS_PC16:
569 *rel->addr |= build_bimm(laddr - (raddr + 4));
570 break;
571
572 default:
573 panic("Unsupported TLB synthesizer relocation %d",
574 rel->type);
575 }
576}
577
578static __init void resolve_relocs(struct reloc *rel, struct label *lab)
579{
580 struct label *l;
581
582 for (; rel->lab != label_invalid; rel++)
583 for (l = lab; l->lab != label_invalid; l++)
584 if (rel->lab == l->lab)
585 __resolve_relocs(rel, l);
586}
587
588static __init void move_relocs(struct reloc *rel, u32 *first, u32 *end,
589 long off)
590{
591 for (; rel->lab != label_invalid; rel++)
592 if (rel->addr >= first && rel->addr < end)
593 rel->addr += off;
594}
595
596static __init void move_labels(struct label *lab, u32 *first, u32 *end,
597 long off)
598{
599 for (; lab->lab != label_invalid; lab++)
600 if (lab->addr >= first && lab->addr < end)
601 lab->addr += off;
602}
603
604static __init void copy_handler(struct reloc *rel, struct label *lab,
605 u32 *first, u32 *end, u32 *target)
606{
607 long off = (long)(target - first);
608
609 memcpy(target, first, (end - first) * sizeof(u32));
610
611 move_relocs(rel, first, end, off);
612 move_labels(lab, first, end, off);
613}
614
615static __init int __attribute__((unused)) insn_has_bdelay(struct reloc *rel,
616 u32 *addr)
617{
618 for (; rel->lab != label_invalid; rel++) {
619 if (rel->addr == addr
620 && (rel->type == R_MIPS_PC16
621 || rel->type == R_MIPS_26))
622 return 1;
623 }
624
625 return 0;
626}
627
628/* convenience functions for labeled branches */
629static void __attribute__((unused)) il_bltz(u32 **p, struct reloc **r,
630 unsigned int reg, enum label_id l)
631{
632 r_mips_pc16(r, *p, l);
633 i_bltz(p, reg, 0);
634}
635
636static void __attribute__((unused)) il_b(u32 **p, struct reloc **r,
637 enum label_id l)
638{
639 r_mips_pc16(r, *p, l);
640 i_b(p, 0);
641}
642
643static void il_beqz(u32 **p, struct reloc **r, unsigned int reg,
644 enum label_id l)
645{
646 r_mips_pc16(r, *p, l);
647 i_beqz(p, reg, 0);
648}
649
650static void __attribute__((unused))
651il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
652{
653 r_mips_pc16(r, *p, l);
654 i_beqzl(p, reg, 0);
655}
656
657static void il_bnez(u32 **p, struct reloc **r, unsigned int reg,
658 enum label_id l)
659{
660 r_mips_pc16(r, *p, l);
661 i_bnez(p, reg, 0);
662}
663
664static void il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
665 enum label_id l)
666{
667 r_mips_pc16(r, *p, l);
668 i_bgezl(p, reg, 0);
669}
670
671/* The only general purpose registers allowed in TLB handlers. */
672#define K0 26
673#define K1 27
674
675/* Some CP0 registers */
676#define C0_INDEX 0
677#define C0_ENTRYLO0 2
678#define C0_ENTRYLO1 3
679#define C0_CONTEXT 4
680#define C0_BADVADDR 8
681#define C0_ENTRYHI 10
682#define C0_EPC 14
683#define C0_XCONTEXT 20
684
685#ifdef CONFIG_MIPS64
686# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT)
687#else
688# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT)
689#endif
690
691/* The worst case length of the handler is around 18 instructions for
692 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
693 * Maximum space available is 32 instructions for R3000 and 64
694 * instructions for R4000.
695 *
696 * We deliberately chose a buffer size of 128, so we won't scribble
697 * over anything important on overflow before we panic.
698 */
699static __initdata u32 tlb_handler[128];
700
701/* simply assume worst case size for labels and relocs */
702static __initdata struct label labels[128];
703static __initdata struct reloc relocs[128];
704
705/*
706 * The R3000 TLB handler is simple.
707 */
708static void __init build_r3000_tlb_refill_handler(void)
709{
710 long pgdc = (long)pgd_current;
711 u32 *p;
712
713 memset(tlb_handler, 0, sizeof(tlb_handler));
714 p = tlb_handler;
715
716 i_mfc0(&p, K0, C0_BADVADDR);
717 i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */
718 i_lw(&p, K1, rel_lo(pgdc), K1);
719 i_srl(&p, K0, K0, 22); /* load delay */
720 i_sll(&p, K0, K0, 2);
721 i_addu(&p, K1, K1, K0);
722 i_mfc0(&p, K0, C0_CONTEXT);
723 i_lw(&p, K1, 0, K1); /* cp0 delay */
724 i_andi(&p, K0, K0, 0xffc); /* load delay */
725 i_addu(&p, K1, K1, K0);
726 i_lw(&p, K0, 0, K1);
727 i_nop(&p); /* load delay */
728 i_mtc0(&p, K0, C0_ENTRYLO0);
729 i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
730 i_tlbwr(&p); /* cp0 delay */
731 i_jr(&p, K1);
732 i_rfe(&p); /* branch delay */
733
734 if (p > tlb_handler + 32)
735 panic("TLB refill handler space exceeded");
736
737 printk("Synthesized TLB handler (%u instructions).\n",
738 (unsigned int)(p - tlb_handler));
739#ifdef DEBUG_TLB
740 {
741 int i;
742
743 for (i = 0; i < (p - tlb_handler); i++)
744 printk("%08x\n", tlb_handler[i]);
745 }
746#endif
747
748 memcpy((void *)CAC_BASE, tlb_handler, 0x80);
749 flush_icache_range(CAC_BASE, CAC_BASE + 0x80);
750}
751
752/*
753 * The R4000 TLB handler is much more complicated. We have two
754 * consecutive handler areas with 32 instructions space each.
755 * Since they aren't used at the same time, we can overflow in the
756 * other one.To keep things simple, we first assume linear space,
757 * then we relocate it to the final handler layout as needed.
758 */
759static __initdata u32 final_handler[64];
760
761/*
762 * Hazards
763 *
764 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
765 * 2. A timing hazard exists for the TLBP instruction.
766 *
767 * stalling_instruction
768 * TLBP
769 *
770 * The JTLB is being read for the TLBP throughout the stall generated by the
771 * previous instruction. This is not really correct as the stalling instruction
772 * can modify the address used to access the JTLB. The failure symptom is that
773 * the TLBP instruction will use an address created for the stalling instruction
774 * and not the address held in C0_ENHI and thus report the wrong results.
775 *
776 * The software work-around is to not allow the instruction preceding the TLBP
777 * to stall - make it an NOP or some other instruction guaranteed not to stall.
778 *
779 * Errata 2 will not be fixed. This errata is also on the R5000.
780 *
781 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
782 */
783static __init void __attribute__((unused)) build_tlb_probe_entry(u32 **p)
784{
785 switch (current_cpu_data.cputype) {
786 case CPU_R5000:
787 case CPU_R5000A:
788 case CPU_NEVADA:
789 i_nop(p);
790 i_tlbp(p);
791 break;
792
793 default:
794 i_tlbp(p);
795 break;
796 }
797}
798
799/*
800 * Write random or indexed TLB entry, and care about the hazards from
801 * the preceeding mtc0 and for the following eret.
802 */
803enum tlb_write_entry { tlb_random, tlb_indexed };
804
805static __init void build_tlb_write_entry(u32 **p, struct label **l,
806 struct reloc **r,
807 enum tlb_write_entry wmode)
808{
809 void(*tlbw)(u32 **) = NULL;
810
811 switch (wmode) {
812 case tlb_random: tlbw = i_tlbwr; break;
813 case tlb_indexed: tlbw = i_tlbwi; break;
814 }
815
816 switch (current_cpu_data.cputype) {
817 case CPU_R4000PC:
818 case CPU_R4000SC:
819 case CPU_R4000MC:
820 case CPU_R4400PC:
821 case CPU_R4400SC:
822 case CPU_R4400MC:
823 /*
824 * This branch uses up a mtc0 hazard nop slot and saves
825 * two nops after the tlbw instruction.
826 */
827 il_bgezl(p, r, 0, label_tlbw_hazard);
828 tlbw(p);
829 l_tlbw_hazard(l, *p);
830 i_nop(p);
831 break;
832
833 case CPU_R4600:
834 case CPU_R4700:
835 case CPU_R5000:
836 case CPU_R5000A:
837 case CPU_5KC:
838 case CPU_TX49XX:
839 case CPU_AU1000:
840 case CPU_AU1100:
841 case CPU_AU1500:
842 case CPU_AU1550:
843 i_nop(p);
844 tlbw(p);
845 break;
846
847 case CPU_R10000:
848 case CPU_R12000:
849 case CPU_4KC:
850 case CPU_SB1:
851 case CPU_4KSC:
852 case CPU_20KC:
853 case CPU_25KF:
854 tlbw(p);
855 break;
856
857 case CPU_NEVADA:
858 i_nop(p); /* QED specifies 2 nops hazard */
859 /*
860 * This branch uses up a mtc0 hazard nop slot and saves
861 * a nop after the tlbw instruction.
862 */
863 il_bgezl(p, r, 0, label_tlbw_hazard);
864 tlbw(p);
865 l_tlbw_hazard(l, *p);
866 break;
867
868 case CPU_RM7000:
869 i_nop(p);
870 i_nop(p);
871 i_nop(p);
872 i_nop(p);
873 tlbw(p);
874 break;
875
876 case CPU_4KEC:
877 case CPU_24K:
878 i_ehb(p);
879 tlbw(p);
880 break;
881
882 case CPU_RM9000:
883 /*
884 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
885 * use of the JTLB for instructions should not occur for 4
886 * cpu cycles and use for data translations should not occur
887 * for 3 cpu cycles.
888 */
889 i_ssnop(p);
890 i_ssnop(p);
891 i_ssnop(p);
892 i_ssnop(p);
893 tlbw(p);
894 i_ssnop(p);
895 i_ssnop(p);
896 i_ssnop(p);
897 i_ssnop(p);
898 break;
899
900 case CPU_VR4111:
901 case CPU_VR4121:
902 case CPU_VR4122:
903 case CPU_VR4181:
904 case CPU_VR4181A:
905 i_nop(p);
906 i_nop(p);
907 tlbw(p);
908 i_nop(p);
909 i_nop(p);
910 break;
911
912 case CPU_VR4131:
913 case CPU_VR4133:
914 i_nop(p);
915 i_nop(p);
916 tlbw(p);
917 break;
918
919 default:
920 panic("No TLB refill handler yet (CPU type: %d)",
921 current_cpu_data.cputype);
922 break;
923 }
924}
925
926#ifdef CONFIG_MIPS64
927/*
928 * TMP and PTR are scratch.
929 * TMP will be clobbered, PTR will hold the pmd entry.
930 */
931static __init void
932build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
933 unsigned int tmp, unsigned int ptr)
934{
935 long pgdc = (long)pgd_current;
936
937 /*
938 * The vmalloc handling is not in the hotpath.
939 */
940 i_dmfc0(p, tmp, C0_BADVADDR);
941 il_bltz(p, r, tmp, label_vmalloc);
942 /* No i_nop needed here, since the next insn doesn't touch TMP. */
943
944#ifdef CONFIG_SMP
945 /*
946 * 64 bit SMP has the lower part of &pgd_current[smp_processor_id()]
947 * stored in CONTEXT.
948 */
949 if (in_compat_space_p(pgdc)) {
950 i_dmfc0(p, ptr, C0_CONTEXT);
951 i_dsra(p, ptr, ptr, 23);
952 i_ld(p, ptr, 0, ptr);
953 } else {
954#ifdef CONFIG_BUILD_ELF64
955 i_dmfc0(p, ptr, C0_CONTEXT);
956 i_dsrl(p, ptr, ptr, 23);
957 i_dsll(p, ptr, ptr, 3);
958 i_LA_mostly(p, tmp, pgdc);
959 i_daddu(p, ptr, ptr, tmp);
960 i_dmfc0(p, tmp, C0_BADVADDR);
961 i_ld(p, ptr, rel_lo(pgdc), ptr);
962#else
963 i_dmfc0(p, ptr, C0_CONTEXT);
964 i_lui(p, tmp, rel_highest(pgdc));
965 i_dsll(p, ptr, ptr, 9);
966 i_daddiu(p, tmp, tmp, rel_higher(pgdc));
967 i_dsrl32(p, ptr, ptr, 0);
968 i_and(p, ptr, ptr, tmp);
969 i_dmfc0(p, tmp, C0_BADVADDR);
970 i_ld(p, ptr, 0, ptr);
971#endif
972 }
973#else
974 i_LA_mostly(p, ptr, pgdc);
975 i_ld(p, ptr, rel_lo(pgdc), ptr);
976#endif
977
978 l_vmalloc_done(l, *p);
979 i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3); /* get pgd offset in bytes */
980 i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
981 i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
982 i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
983 i_ld(p, ptr, 0, ptr); /* get pmd pointer */
984 i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
985 i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
986 i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
987}
988
989/*
990 * BVADDR is the faulting address, PTR is scratch.
991 * PTR will hold the pgd for vmalloc.
992 */
993static __init void
994build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
995 unsigned int bvaddr, unsigned int ptr)
996{
997 long swpd = (long)swapper_pg_dir;
998
999 l_vmalloc(l, *p);
1000 i_LA(p, ptr, VMALLOC_START);
1001 i_dsubu(p, bvaddr, bvaddr, ptr);
1002
1003 if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
1004 il_b(p, r, label_vmalloc_done);
1005 i_lui(p, ptr, rel_hi(swpd));
1006 } else {
1007 i_LA_mostly(p, ptr, swpd);
1008 il_b(p, r, label_vmalloc_done);
1009 i_daddiu(p, ptr, ptr, rel_lo(swpd));
1010 }
1011}
1012
1013#else /* !CONFIG_MIPS64 */
1014
1015/*
1016 * TMP and PTR are scratch.
1017 * TMP will be clobbered, PTR will hold the pgd entry.
1018 */
1019static __init void __attribute__((unused))
1020build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
1021{
1022 long pgdc = (long)pgd_current;
1023
1024 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
1025#ifdef CONFIG_SMP
1026 i_mfc0(p, ptr, C0_CONTEXT);
1027 i_LA_mostly(p, tmp, pgdc);
1028 i_srl(p, ptr, ptr, 23);
1029 i_sll(p, ptr, ptr, 2);
1030 i_addu(p, ptr, tmp, ptr);
1031#else
1032 i_LA_mostly(p, ptr, pgdc);
1033#endif
1034 i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
1035 i_lw(p, ptr, rel_lo(pgdc), ptr);
1036 i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
1037 i_sll(p, tmp, tmp, PGD_T_LOG2);
1038 i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
1039}
1040
1041#endif /* !CONFIG_MIPS64 */
1042
1043static __init void build_adjust_context(u32 **p, unsigned int ctx)
1044{
1045 unsigned int shift = 4 - (PTE_T_LOG2 + 1);
1046 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
1047
1048 switch (current_cpu_data.cputype) {
1049 case CPU_VR41XX:
1050 case CPU_VR4111:
1051 case CPU_VR4121:
1052 case CPU_VR4122:
1053 case CPU_VR4131:
1054 case CPU_VR4181:
1055 case CPU_VR4181A:
1056 case CPU_VR4133:
1057 shift += 2;
1058 break;
1059
1060 default:
1061 break;
1062 }
1063
1064 if (shift)
1065 i_SRL(p, ctx, ctx, shift);
1066 i_andi(p, ctx, ctx, mask);
1067}
1068
1069static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
1070{
1071 /*
1072 * Bug workaround for the Nevada. It seems as if under certain
1073 * circumstances the move from cp0_context might produce a
1074 * bogus result when the mfc0 instruction and its consumer are
1075 * in a different cacheline or a load instruction, probably any
1076 * memory reference, is between them.
1077 */
1078 switch (current_cpu_data.cputype) {
1079 case CPU_NEVADA:
1080 i_LW(p, ptr, 0, ptr);
1081 GET_CONTEXT(p, tmp); /* get context reg */
1082 break;
1083
1084 default:
1085 GET_CONTEXT(p, tmp); /* get context reg */
1086 i_LW(p, ptr, 0, ptr);
1087 break;
1088 }
1089
1090 build_adjust_context(p, tmp);
1091 i_ADDU(p, ptr, ptr, tmp); /* add in offset */
1092}
1093
1094static __init void build_update_entries(u32 **p, unsigned int tmp,
1095 unsigned int ptep)
1096{
1097 /*
1098 * 64bit address support (36bit on a 32bit CPU) in a 32bit
1099 * Kernel is a special case. Only a few CPUs use it.
1100 */
1101#ifdef CONFIG_64BIT_PHYS_ADDR
1102 if (cpu_has_64bits) {
1103 i_ld(p, tmp, 0, ptep); /* get even pte */
1104 i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
1105 i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
1106 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1107 i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
1108 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1109 } else {
1110 int pte_off_even = sizeof(pte_t) / 2;
1111 int pte_off_odd = pte_off_even + sizeof(pte_t);
1112
1113 /* The pte entries are pre-shifted */
1114 i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
1115 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1116 i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
1117 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1118 }
1119#else
1120 i_LW(p, tmp, 0, ptep); /* get even pte */
1121 i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
1122 if (r45k_bvahwbug())
1123 build_tlb_probe_entry(p);
1124 i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
1125 if (r4k_250MHZhwbug())
1126 i_mtc0(p, 0, C0_ENTRYLO0);
1127 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1128 i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
1129 if (r45k_bvahwbug())
1130 i_mfc0(p, tmp, C0_INDEX);
1131 if (r4k_250MHZhwbug())
1132 i_mtc0(p, 0, C0_ENTRYLO1);
1133 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1134#endif
1135}
1136
1137static void __init build_r4000_tlb_refill_handler(void)
1138{
1139 u32 *p = tlb_handler;
1140 struct label *l = labels;
1141 struct reloc *r = relocs;
1142 u32 *f;
1143 unsigned int final_len;
1144
1145 memset(tlb_handler, 0, sizeof(tlb_handler));
1146 memset(labels, 0, sizeof(labels));
1147 memset(relocs, 0, sizeof(relocs));
1148 memset(final_handler, 0, sizeof(final_handler));
1149
1150 /*
1151 * create the plain linear handler
1152 */
1153 if (bcm1250_m3_war()) {
1154 i_MFC0(&p, K0, C0_BADVADDR);
1155 i_MFC0(&p, K1, C0_ENTRYHI);
1156 i_xor(&p, K0, K0, K1);
1157 i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
1158 il_bnez(&p, &r, K0, label_leave);
1159 /* No need for i_nop */
1160 }
1161
1162#ifdef CONFIG_MIPS64
1163 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
1164#else
1165 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
1166#endif
1167
1168 build_get_ptep(&p, K0, K1);
1169 build_update_entries(&p, K0, K1);
1170 build_tlb_write_entry(&p, &l, &r, tlb_random);
1171 l_leave(&l, p);
1172 i_eret(&p); /* return from trap */
1173
1174#ifdef CONFIG_MIPS64
1175 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
1176#endif
1177
1178 /*
1179 * Overflow check: For the 64bit handler, we need at least one
1180 * free instruction slot for the wrap-around branch. In worst
1181 * case, if the intended insertion point is a delay slot, we
1182 * need three, with the the second nop'ed and the third being
1183 * unused.
1184 */
1185#ifdef CONFIG_MIPS32
1186 if ((p - tlb_handler) > 64)
1187 panic("TLB refill handler space exceeded");
1188#else
1189 if (((p - tlb_handler) > 63)
1190 || (((p - tlb_handler) > 61)
1191 && insn_has_bdelay(relocs, tlb_handler + 29)))
1192 panic("TLB refill handler space exceeded");
1193#endif
1194
1195 /*
1196 * Now fold the handler in the TLB refill handler space.
1197 */
1198#ifdef CONFIG_MIPS32
1199 f = final_handler;
1200 /* Simplest case, just copy the handler. */
1201 copy_handler(relocs, labels, tlb_handler, p, f);
1202 final_len = p - tlb_handler;
1203#else /* CONFIG_MIPS64 */
1204 f = final_handler + 32;
1205 if ((p - tlb_handler) <= 32) {
1206 /* Just copy the handler. */
1207 copy_handler(relocs, labels, tlb_handler, p, f);
1208 final_len = p - tlb_handler;
1209 } else {
1210 u32 *split = tlb_handler + 30;
1211
1212 /*
1213 * Find the split point.
1214 */
1215 if (insn_has_bdelay(relocs, split - 1))
1216 split--;
1217
1218 /* Copy first part of the handler. */
1219 copy_handler(relocs, labels, tlb_handler, split, f);
1220 f += split - tlb_handler;
1221
1222 /* Insert branch. */
1223 l_split(&l, final_handler);
1224 il_b(&f, &r, label_split);
1225 if (insn_has_bdelay(relocs, split))
1226 i_nop(&f);
1227 else {
1228 copy_handler(relocs, labels, split, split + 1, f);
1229 move_labels(labels, f, f + 1, -1);
1230 f++;
1231 split++;
1232 }
1233
1234 /* Copy the rest of the handler. */
1235 copy_handler(relocs, labels, split, p, final_handler);
1236 final_len = (f - (final_handler + 32)) + (p - split);
1237 }
1238#endif /* CONFIG_MIPS64 */
1239
1240 resolve_relocs(relocs, labels);
1241 printk("Synthesized TLB refill handler (%u instructions).\n",
1242 final_len);
1243
1244#ifdef DEBUG_TLB
1245 {
1246 int i;
1247
1248 for (i = 0; i < 64; i++)
1249 printk("%08x\n", final_handler[i]);
1250 }
1251#endif
1252
1253 memcpy((void *)CAC_BASE, final_handler, 0x100);
1254 flush_icache_range(CAC_BASE, CAC_BASE + 0x100);
1255}
1256
1257/*
1258 * TLB load/store/modify handlers.
1259 *
1260 * Only the fastpath gets synthesized at runtime, the slowpath for
1261 * do_page_fault remains normal asm.
1262 */
1263extern void tlb_do_page_fault_0(void);
1264extern void tlb_do_page_fault_1(void);
1265
1266#define __tlb_handler_align \
1267 __attribute__((__aligned__(1 << CONFIG_MIPS_L1_CACHE_SHIFT)))
1268
1269/*
1270 * 128 instructions for the fastpath handler is generous and should
1271 * never be exceeded.
1272 */
1273#define FASTPATH_SIZE 128
1274
1275u32 __tlb_handler_align handle_tlbl[FASTPATH_SIZE];
1276u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE];
1277u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE];
1278
1279static void __init
1280iPTE_LW(u32 **p, struct label **l, unsigned int pte, int offset,
1281 unsigned int ptr)
1282{
1283#ifdef CONFIG_SMP
1284# ifdef CONFIG_64BIT_PHYS_ADDR
1285 if (cpu_has_64bits)
1286 i_lld(p, pte, offset, ptr);
1287 else
1288# endif
1289 i_LL(p, pte, offset, ptr);
1290#else
1291# ifdef CONFIG_64BIT_PHYS_ADDR
1292 if (cpu_has_64bits)
1293 i_ld(p, pte, offset, ptr);
1294 else
1295# endif
1296 i_LW(p, pte, offset, ptr);
1297#endif
1298}
1299
1300static void __init
1301iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset,
1302 unsigned int ptr)
1303{
1304#ifdef CONFIG_SMP
1305# ifdef CONFIG_64BIT_PHYS_ADDR
1306 if (cpu_has_64bits)
1307 i_scd(p, pte, offset, ptr);
1308 else
1309# endif
1310 i_SC(p, pte, offset, ptr);
1311
1312 if (r10000_llsc_war())
1313 il_beqzl(p, r, pte, label_smp_pgtable_change);
1314 else
1315 il_beqz(p, r, pte, label_smp_pgtable_change);
1316
1317# ifdef CONFIG_64BIT_PHYS_ADDR
1318 if (!cpu_has_64bits) {
1319 /* no i_nop needed */
1320 i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1321 i_ori(p, pte, pte, _PAGE_VALID);
1322 i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1323 il_beqz(p, r, pte, label_smp_pgtable_change);
1324 /* no i_nop needed */
1325 i_lw(p, pte, 0, ptr);
1326 } else
1327 i_nop(p);
1328# else
1329 i_nop(p);
1330# endif
1331#else
1332# ifdef CONFIG_64BIT_PHYS_ADDR
1333 if (cpu_has_64bits)
1334 i_sd(p, pte, offset, ptr);
1335 else
1336# endif
1337 i_SW(p, pte, offset, ptr);
1338
1339# ifdef CONFIG_64BIT_PHYS_ADDR
1340 if (!cpu_has_64bits) {
1341 i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1342 i_ori(p, pte, pte, _PAGE_VALID);
1343 i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1344 i_lw(p, pte, 0, ptr);
1345 }
1346# endif
1347#endif
1348}
1349
1350/*
1351 * Check if PTE is present, if not then jump to LABEL. PTR points to
1352 * the page table where this PTE is located, PTE will be re-loaded
1353 * with it's original value.
1354 */
1355static void __init
1356build_pte_present(u32 **p, struct label **l, struct reloc **r,
1357 unsigned int pte, unsigned int ptr, enum label_id lid)
1358{
1359 i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1360 i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1361 il_bnez(p, r, pte, lid);
1362 iPTE_LW(p, l, pte, 0, ptr);
1363}
1364
1365/* Make PTE valid, store result in PTR. */
1366static void __init
1367build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
1368 unsigned int ptr)
1369{
1370 i_ori(p, pte, pte, _PAGE_VALID | _PAGE_ACCESSED);
1371 iPTE_SW(p, r, pte, 0, ptr);
1372}
1373
1374/*
1375 * Check if PTE can be written to, if not branch to LABEL. Regardless
1376 * restore PTE with value from PTR when done.
1377 */
1378static void __init
1379build_pte_writable(u32 **p, struct label **l, struct reloc **r,
1380 unsigned int pte, unsigned int ptr, enum label_id lid)
1381{
1382 i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1383 i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1384 il_bnez(p, r, pte, lid);
1385 iPTE_LW(p, l, pte, 0, ptr);
1386}
1387
1388/* Make PTE writable, update software status bits as well, then store
1389 * at PTR.
1390 */
1391static void __init
1392build_make_write(u32 **p, struct reloc **r, unsigned int pte,
1393 unsigned int ptr)
1394{
1395 i_ori(p, pte, pte,
1396 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
1397 iPTE_SW(p, r, pte, 0, ptr);
1398}
1399
1400/*
1401 * Check if PTE can be modified, if not branch to LABEL. Regardless
1402 * restore PTE with value from PTR when done.
1403 */
1404static void __init
1405build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
1406 unsigned int pte, unsigned int ptr, enum label_id lid)
1407{
1408 i_andi(p, pte, pte, _PAGE_WRITE);
1409 il_beqz(p, r, pte, lid);
1410 iPTE_LW(p, l, pte, 0, ptr);
1411}
1412
1413/*
1414 * R3000 style TLB load/store/modify handlers.
1415 */
1416
1417/* This places the pte in the page table at PTR into ENTRYLO0. */
1418static void __init
1419build_r3000_pte_reload(u32 **p, unsigned int ptr)
1420{
1421 i_lw(p, ptr, 0, ptr);
1422 i_nop(p); /* load delay */
1423 i_mtc0(p, ptr, C0_ENTRYLO0);
1424 i_nop(p); /* cp0 delay */
1425}
1426
1427/*
1428 * The index register may have the probe fail bit set,
1429 * because we would trap on access kseg2, i.e. without refill.
1430 */
1431static void __init
1432build_r3000_tlb_write(u32 **p, struct label **l, struct reloc **r,
1433 unsigned int tmp)
1434{
1435 i_mfc0(p, tmp, C0_INDEX);
1436 i_nop(p); /* cp0 delay */
1437 il_bltz(p, r, tmp, label_r3000_write_probe_fail);
1438 i_nop(p); /* branch delay */
1439 i_tlbwi(p);
1440 il_b(p, r, label_r3000_write_probe_ok);
1441 i_nop(p); /* branch delay */
1442 l_r3000_write_probe_fail(l, *p);
1443 i_tlbwr(p);
1444 l_r3000_write_probe_ok(l, *p);
1445}
1446
1447static void __init
1448build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1449 unsigned int ptr)
1450{
1451 long pgdc = (long)pgd_current;
1452
1453 i_mfc0(p, pte, C0_BADVADDR);
1454 i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */
1455 i_lw(p, ptr, rel_lo(pgdc), ptr);
1456 i_srl(p, pte, pte, 22); /* load delay */
1457 i_sll(p, pte, pte, 2);
1458 i_addu(p, ptr, ptr, pte);
1459 i_mfc0(p, pte, C0_CONTEXT);
1460 i_lw(p, ptr, 0, ptr); /* cp0 delay */
1461 i_andi(p, pte, pte, 0xffc); /* load delay */
1462 i_addu(p, ptr, ptr, pte);
1463 i_lw(p, pte, 0, ptr);
1464 i_nop(p); /* load delay */
1465 i_tlbp(p);
1466}
1467
1468static void __init
1469build_r3000_tlbchange_handler_tail(u32 **p, unsigned int tmp)
1470{
1471 i_mfc0(p, tmp, C0_EPC);
1472 i_nop(p); /* cp0 delay */
1473 i_jr(p, tmp);
1474 i_rfe(p); /* branch delay */
1475}
1476
1477static void __init build_r3000_tlb_load_handler(void)
1478{
1479 u32 *p = handle_tlbl;
1480 struct label *l = labels;
1481 struct reloc *r = relocs;
1482
1483 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1484 memset(labels, 0, sizeof(labels));
1485 memset(relocs, 0, sizeof(relocs));
1486
1487 build_r3000_tlbchange_handler_head(&p, K0, K1);
1488 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1489 build_make_valid(&p, &r, K0, K1);
1490 build_r3000_pte_reload(&p, K1);
1491 build_r3000_tlb_write(&p, &l, &r, K0);
1492 build_r3000_tlbchange_handler_tail(&p, K0);
1493
1494 l_nopage_tlbl(&l, p);
1495 i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1496 i_nop(&p);
1497
1498 if ((p - handle_tlbl) > FASTPATH_SIZE)
1499 panic("TLB load handler fastpath space exceeded");
1500
1501 resolve_relocs(relocs, labels);
1502 printk("Synthesized TLB load handler fastpath (%u instructions).\n",
1503 (unsigned int)(p - handle_tlbl));
1504
1505#ifdef DEBUG_TLB
1506 {
1507 int i;
1508
1509 for (i = 0; i < FASTPATH_SIZE; i++)
1510 printk("%08x\n", handle_tlbl[i]);
1511 }
1512#endif
1513
1514 flush_icache_range((unsigned long)handle_tlbl,
1515 (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32));
1516}
1517
1518static void __init build_r3000_tlb_store_handler(void)
1519{
1520 u32 *p = handle_tlbs;
1521 struct label *l = labels;
1522 struct reloc *r = relocs;
1523
1524 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1525 memset(labels, 0, sizeof(labels));
1526 memset(relocs, 0, sizeof(relocs));
1527
1528 build_r3000_tlbchange_handler_head(&p, K0, K1);
1529 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1530 build_make_write(&p, &r, K0, K1);
1531 build_r3000_pte_reload(&p, K1);
1532 build_r3000_tlb_write(&p, &l, &r, K0);
1533 build_r3000_tlbchange_handler_tail(&p, K0);
1534
1535 l_nopage_tlbs(&l, p);
1536 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1537 i_nop(&p);
1538
1539 if ((p - handle_tlbs) > FASTPATH_SIZE)
1540 panic("TLB store handler fastpath space exceeded");
1541
1542 resolve_relocs(relocs, labels);
1543 printk("Synthesized TLB store handler fastpath (%u instructions).\n",
1544 (unsigned int)(p - handle_tlbs));
1545
1546#ifdef DEBUG_TLB
1547 {
1548 int i;
1549
1550 for (i = 0; i < FASTPATH_SIZE; i++)
1551 printk("%08x\n", handle_tlbs[i]);
1552 }
1553#endif
1554
1555 flush_icache_range((unsigned long)handle_tlbs,
1556 (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32));
1557}
1558
1559static void __init build_r3000_tlb_modify_handler(void)
1560{
1561 u32 *p = handle_tlbm;
1562 struct label *l = labels;
1563 struct reloc *r = relocs;
1564
1565 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1566 memset(labels, 0, sizeof(labels));
1567 memset(relocs, 0, sizeof(relocs));
1568
1569 build_r3000_tlbchange_handler_head(&p, K0, K1);
1570 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1571 build_make_write(&p, &r, K0, K1);
1572 build_r3000_pte_reload(&p, K1);
1573 i_tlbwi(&p);
1574 build_r3000_tlbchange_handler_tail(&p, K0);
1575
1576 l_nopage_tlbm(&l, p);
1577 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1578 i_nop(&p);
1579
1580 if ((p - handle_tlbm) > FASTPATH_SIZE)
1581 panic("TLB modify handler fastpath space exceeded");
1582
1583 resolve_relocs(relocs, labels);
1584 printk("Synthesized TLB modify handler fastpath (%u instructions).\n",
1585 (unsigned int)(p - handle_tlbm));
1586
1587#ifdef DEBUG_TLB
1588 {
1589 int i;
1590
1591 for (i = 0; i < FASTPATH_SIZE; i++)
1592 printk("%08x\n", handle_tlbm[i]);
1593 }
1594#endif
1595
1596 flush_icache_range((unsigned long)handle_tlbm,
1597 (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32));
1598}
1599
1600/*
1601 * R4000 style TLB load/store/modify handlers.
1602 */
1603static void __init
1604build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
1605 struct reloc **r, unsigned int pte,
1606 unsigned int ptr)
1607{
1608#ifdef CONFIG_MIPS64
1609 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
1610#else
1611 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
1612#endif
1613
1614 i_MFC0(p, pte, C0_BADVADDR);
1615 i_LW(p, ptr, 0, ptr);
1616 i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
1617 i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
1618 i_ADDU(p, ptr, ptr, pte);
1619
1620#ifdef CONFIG_SMP
1621 l_smp_pgtable_change(l, *p);
1622# endif
1623 iPTE_LW(p, l, pte, 0, ptr); /* get even pte */
1624 build_tlb_probe_entry(p);
1625}
1626
1627static void __init
1628build_r4000_tlbchange_handler_tail(u32 **p, struct label **l,
1629 struct reloc **r, unsigned int tmp,
1630 unsigned int ptr)
1631{
1632 i_ori(p, ptr, ptr, sizeof(pte_t));
1633 i_xori(p, ptr, ptr, sizeof(pte_t));
1634 build_update_entries(p, tmp, ptr);
1635 build_tlb_write_entry(p, l, r, tlb_indexed);
1636 l_leave(l, *p);
1637 i_eret(p); /* return from trap */
1638
1639#ifdef CONFIG_MIPS64
1640 build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
1641#endif
1642}
1643
1644static void __init build_r4000_tlb_load_handler(void)
1645{
1646 u32 *p = handle_tlbl;
1647 struct label *l = labels;
1648 struct reloc *r = relocs;
1649
1650 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1651 memset(labels, 0, sizeof(labels));
1652 memset(relocs, 0, sizeof(relocs));
1653
1654 if (bcm1250_m3_war()) {
1655 i_MFC0(&p, K0, C0_BADVADDR);
1656 i_MFC0(&p, K1, C0_ENTRYHI);
1657 i_xor(&p, K0, K0, K1);
1658 i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
1659 il_bnez(&p, &r, K0, label_leave);
1660 /* No need for i_nop */
1661 }
1662
1663 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1664 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1665 build_make_valid(&p, &r, K0, K1);
1666 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1667
1668 l_nopage_tlbl(&l, p);
1669 i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1670 i_nop(&p);
1671
1672 if ((p - handle_tlbl) > FASTPATH_SIZE)
1673 panic("TLB load handler fastpath space exceeded");
1674
1675 resolve_relocs(relocs, labels);
1676 printk("Synthesized TLB load handler fastpath (%u instructions).\n",
1677 (unsigned int)(p - handle_tlbl));
1678
1679#ifdef DEBUG_TLB
1680 {
1681 int i;
1682
1683 for (i = 0; i < FASTPATH_SIZE; i++)
1684 printk("%08x\n", handle_tlbl[i]);
1685 }
1686#endif
1687
1688 flush_icache_range((unsigned long)handle_tlbl,
1689 (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32));
1690}
1691
1692static void __init build_r4000_tlb_store_handler(void)
1693{
1694 u32 *p = handle_tlbs;
1695 struct label *l = labels;
1696 struct reloc *r = relocs;
1697
1698 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1699 memset(labels, 0, sizeof(labels));
1700 memset(relocs, 0, sizeof(relocs));
1701
1702 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1703 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1704 build_make_write(&p, &r, K0, K1);
1705 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1706
1707 l_nopage_tlbs(&l, p);
1708 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1709 i_nop(&p);
1710
1711 if ((p - handle_tlbs) > FASTPATH_SIZE)
1712 panic("TLB store handler fastpath space exceeded");
1713
1714 resolve_relocs(relocs, labels);
1715 printk("Synthesized TLB store handler fastpath (%u instructions).\n",
1716 (unsigned int)(p - handle_tlbs));
1717
1718#ifdef DEBUG_TLB
1719 {
1720 int i;
1721
1722 for (i = 0; i < FASTPATH_SIZE; i++)
1723 printk("%08x\n", handle_tlbs[i]);
1724 }
1725#endif
1726
1727 flush_icache_range((unsigned long)handle_tlbs,
1728 (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32));
1729}
1730
1731static void __init build_r4000_tlb_modify_handler(void)
1732{
1733 u32 *p = handle_tlbm;
1734 struct label *l = labels;
1735 struct reloc *r = relocs;
1736
1737 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1738 memset(labels, 0, sizeof(labels));
1739 memset(relocs, 0, sizeof(relocs));
1740
1741 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1742 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1743 /* Present and writable bits set, set accessed and dirty bits. */
1744 build_make_write(&p, &r, K0, K1);
1745 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1746
1747 l_nopage_tlbm(&l, p);
1748 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1749 i_nop(&p);
1750
1751 if ((p - handle_tlbm) > FASTPATH_SIZE)
1752 panic("TLB modify handler fastpath space exceeded");
1753
1754 resolve_relocs(relocs, labels);
1755 printk("Synthesized TLB modify handler fastpath (%u instructions).\n",
1756 (unsigned int)(p - handle_tlbm));
1757
1758#ifdef DEBUG_TLB
1759 {
1760 int i;
1761
1762 for (i = 0; i < FASTPATH_SIZE; i++)
1763 printk("%08x\n", handle_tlbm[i]);
1764 }
1765#endif
1766
1767 flush_icache_range((unsigned long)handle_tlbm,
1768 (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32));
1769}
1770
1771void __init build_tlb_refill_handler(void)
1772{
1773 /*
1774 * The refill handler is generated per-CPU, multi-node systems
1775 * may have local storage for it. The other handlers are only
1776 * needed once.
1777 */
1778 static int run_once = 0;
1779
1780 switch (current_cpu_data.cputype) {
1781 case CPU_R2000:
1782 case CPU_R3000:
1783 case CPU_R3000A:
1784 case CPU_R3081E:
1785 case CPU_TX3912:
1786 case CPU_TX3922:
1787 case CPU_TX3927:
1788 build_r3000_tlb_refill_handler();
1789 if (!run_once) {
1790 build_r3000_tlb_load_handler();
1791 build_r3000_tlb_store_handler();
1792 build_r3000_tlb_modify_handler();
1793 run_once++;
1794 }
1795 break;
1796
1797 case CPU_R6000:
1798 case CPU_R6000A:
1799 panic("No R6000 TLB refill handler yet");
1800 break;
1801
1802 case CPU_R8000:
1803 panic("No R8000 TLB refill handler yet");
1804 break;
1805
1806 default:
1807 build_r4000_tlb_refill_handler();
1808 if (!run_once) {
1809 build_r4000_tlb_load_handler();
1810 build_r4000_tlb_store_handler();
1811 build_r4000_tlb_modify_handler();
1812 run_once++;
1813 }
1814 }
1815}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-01 18:00:02 -0500