aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/mm
diff options
context:
space:
mode:
authorSage Weil <sage@inktank.com>2013-08-15 14:11:45 -0400
committerSage Weil <sage@inktank.com>2013-08-15 14:11:45 -0400
commitee3e542fec6e69bc9fb668698889a37d93950ddf (patch)
treee74ee766a4764769ef1d3d45d266b4dea64101d3 /arch/mips/mm
parentfe2a801b50c0bb8039d627e5ae1fec249d10ff39 (diff)
parentf1d6e17f540af37bb1891480143669ba7636c4cf (diff)
Merge remote-tracking branch 'linus/master' into testing
Diffstat (limited to 'arch/mips/mm')
-rw-r--r--arch/mips/mm/Makefile2
-rw-r--r--arch/mips/mm/c-octeon.c6
-rw-r--r--arch/mips/mm/c-r3k.c8
-rw-r--r--arch/mips/mm/c-r4k.c34
-rw-r--r--arch/mips/mm/c-tx39.c2
-rw-r--r--arch/mips/mm/cache.c2
-rw-r--r--arch/mips/mm/cerr-sb1.c4
-rw-r--r--arch/mips/mm/cex-sb1.S4
-rw-r--r--arch/mips/mm/dma-default.c3
-rw-r--r--arch/mips/mm/fault.c15
-rw-r--r--arch/mips/mm/init.c62
-rw-r--r--arch/mips/mm/mmap.c2
-rw-r--r--arch/mips/mm/page.c42
-rw-r--r--arch/mips/mm/sc-ip22.c2
-rw-r--r--arch/mips/mm/sc-mips.c2
-rw-r--r--arch/mips/mm/sc-r5k.c2
-rw-r--r--arch/mips/mm/sc-rm7k.c12
-rw-r--r--arch/mips/mm/tlb-funcs.S37
-rw-r--r--arch/mips/mm/tlb-r3k.c2
-rw-r--r--arch/mips/mm/tlb-r4k.c4
-rw-r--r--arch/mips/mm/tlb-r8k.c4
-rw-r--r--arch/mips/mm/tlbex.c327
-rw-r--r--arch/mips/mm/uasm-micromips.c10
-rw-r--r--arch/mips/mm/uasm-mips.c10
-rw-r--r--arch/mips/mm/uasm.c106
25 files changed, 376 insertions, 328 deletions
diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile
index e87aae1f2e80..7f4f93ab22b7 100644
--- a/arch/mips/mm/Makefile
+++ b/arch/mips/mm/Makefile
@@ -4,7 +4,7 @@
4 4
5obj-y += cache.o dma-default.o extable.o fault.o \ 5obj-y += cache.o dma-default.o extable.o fault.o \
6 gup.o init.o mmap.o page.o page-funcs.o \ 6 gup.o init.o mmap.o page.o page-funcs.o \
7 tlbex.o tlbex-fault.o uasm-mips.o 7 tlbex.o tlbex-fault.o tlb-funcs.o uasm-mips.o
8 8
9obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o 9obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o
10obj-$(CONFIG_64BIT) += pgtable-64.o 10obj-$(CONFIG_64BIT) += pgtable-64.o
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index 8557fb552863..a0bcdbb81d41 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -180,7 +180,7 @@ static void octeon_flush_kernel_vmap_range(unsigned long vaddr, int size)
180 * Probe Octeon's caches 180 * Probe Octeon's caches
181 * 181 *
182 */ 182 */
183static void __cpuinit probe_octeon(void) 183static void probe_octeon(void)
184{ 184{
185 unsigned long icache_size; 185 unsigned long icache_size;
186 unsigned long dcache_size; 186 unsigned long dcache_size;
@@ -251,7 +251,7 @@ static void __cpuinit probe_octeon(void)
251 } 251 }
252} 252}
253 253
254static void __cpuinit octeon_cache_error_setup(void) 254static void octeon_cache_error_setup(void)
255{ 255{
256 extern char except_vec2_octeon; 256 extern char except_vec2_octeon;
257 set_handler(0x100, &except_vec2_octeon, 0x80); 257 set_handler(0x100, &except_vec2_octeon, 0x80);
@@ -261,7 +261,7 @@ static void __cpuinit octeon_cache_error_setup(void)
261 * Setup the Octeon cache flush routines 261 * Setup the Octeon cache flush routines
262 * 262 *
263 */ 263 */
264void __cpuinit octeon_cache_init(void) 264void octeon_cache_init(void)
265{ 265{
266 probe_octeon(); 266 probe_octeon();
267 267
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
index 704dc735a59d..2fcde0c8ea02 100644
--- a/arch/mips/mm/c-r3k.c
+++ b/arch/mips/mm/c-r3k.c
@@ -26,7 +26,7 @@
26static unsigned long icache_size, dcache_size; /* Size in bytes */ 26static unsigned long icache_size, dcache_size; /* Size in bytes */
27static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */ 27static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */
28 28
29unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags) 29unsigned long r3k_cache_size(unsigned long ca_flags)
30{ 30{
31 unsigned long flags, status, dummy, size; 31 unsigned long flags, status, dummy, size;
32 volatile unsigned long *p; 32 volatile unsigned long *p;
@@ -61,7 +61,7 @@ unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags)
61 return size * sizeof(*p); 61 return size * sizeof(*p);
62} 62}
63 63
64unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags) 64unsigned long r3k_cache_lsize(unsigned long ca_flags)
65{ 65{
66 unsigned long flags, status, lsize, i; 66 unsigned long flags, status, lsize, i;
67 volatile unsigned long *p; 67 volatile unsigned long *p;
@@ -90,7 +90,7 @@ unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags)
90 return lsize * sizeof(*p); 90 return lsize * sizeof(*p);
91} 91}
92 92
93static void __cpuinit r3k_probe_cache(void) 93static void r3k_probe_cache(void)
94{ 94{
95 dcache_size = r3k_cache_size(ST0_ISC); 95 dcache_size = r3k_cache_size(ST0_ISC);
96 if (dcache_size) 96 if (dcache_size)
@@ -312,7 +312,7 @@ static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size)
312 r3k_flush_dcache_range(start, start + size); 312 r3k_flush_dcache_range(start, start + size);
313} 313}
314 314
315void __cpuinit r3k_cache_init(void) 315void r3k_cache_init(void)
316{ 316{
317 extern void build_clear_page(void); 317 extern void build_clear_page(void);
318 extern void build_copy_page(void); 318 extern void build_copy_page(void);
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index 21813beec7a5..f749f687ee87 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -107,7 +107,7 @@ static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
107 blast_dcache64_page(addr); 107 blast_dcache64_page(addr);
108} 108}
109 109
110static void __cpuinit r4k_blast_dcache_page_setup(void) 110static void r4k_blast_dcache_page_setup(void)
111{ 111{
112 unsigned long dc_lsize = cpu_dcache_line_size(); 112 unsigned long dc_lsize = cpu_dcache_line_size();
113 113
@@ -123,7 +123,7 @@ static void __cpuinit r4k_blast_dcache_page_setup(void)
123 123
124static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 124static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
125 125
126static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) 126static void r4k_blast_dcache_page_indexed_setup(void)
127{ 127{
128 unsigned long dc_lsize = cpu_dcache_line_size(); 128 unsigned long dc_lsize = cpu_dcache_line_size();
129 129
@@ -140,7 +140,7 @@ static void __cpuinit r4k_blast_dcache_page_indexed_setup(void)
140void (* r4k_blast_dcache)(void); 140void (* r4k_blast_dcache)(void);
141EXPORT_SYMBOL(r4k_blast_dcache); 141EXPORT_SYMBOL(r4k_blast_dcache);
142 142
143static void __cpuinit r4k_blast_dcache_setup(void) 143static void r4k_blast_dcache_setup(void)
144{ 144{
145 unsigned long dc_lsize = cpu_dcache_line_size(); 145 unsigned long dc_lsize = cpu_dcache_line_size();
146 146
@@ -227,7 +227,7 @@ static inline void tx49_blast_icache32_page_indexed(unsigned long page)
227 227
228static void (* r4k_blast_icache_page)(unsigned long addr); 228static void (* r4k_blast_icache_page)(unsigned long addr);
229 229
230static void __cpuinit r4k_blast_icache_page_setup(void) 230static void r4k_blast_icache_page_setup(void)
231{ 231{
232 unsigned long ic_lsize = cpu_icache_line_size(); 232 unsigned long ic_lsize = cpu_icache_line_size();
233 233
@@ -244,7 +244,7 @@ static void __cpuinit r4k_blast_icache_page_setup(void)
244 244
245static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 245static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
246 246
247static void __cpuinit r4k_blast_icache_page_indexed_setup(void) 247static void r4k_blast_icache_page_indexed_setup(void)
248{ 248{
249 unsigned long ic_lsize = cpu_icache_line_size(); 249 unsigned long ic_lsize = cpu_icache_line_size();
250 250
@@ -269,7 +269,7 @@ static void __cpuinit r4k_blast_icache_page_indexed_setup(void)
269void (* r4k_blast_icache)(void); 269void (* r4k_blast_icache)(void);
270EXPORT_SYMBOL(r4k_blast_icache); 270EXPORT_SYMBOL(r4k_blast_icache);
271 271
272static void __cpuinit r4k_blast_icache_setup(void) 272static void r4k_blast_icache_setup(void)
273{ 273{
274 unsigned long ic_lsize = cpu_icache_line_size(); 274 unsigned long ic_lsize = cpu_icache_line_size();
275 275
@@ -290,7 +290,7 @@ static void __cpuinit r4k_blast_icache_setup(void)
290 290
291static void (* r4k_blast_scache_page)(unsigned long addr); 291static void (* r4k_blast_scache_page)(unsigned long addr);
292 292
293static void __cpuinit r4k_blast_scache_page_setup(void) 293static void r4k_blast_scache_page_setup(void)
294{ 294{
295 unsigned long sc_lsize = cpu_scache_line_size(); 295 unsigned long sc_lsize = cpu_scache_line_size();
296 296
@@ -308,7 +308,7 @@ static void __cpuinit r4k_blast_scache_page_setup(void)
308 308
309static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 309static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
310 310
311static void __cpuinit r4k_blast_scache_page_indexed_setup(void) 311static void r4k_blast_scache_page_indexed_setup(void)
312{ 312{
313 unsigned long sc_lsize = cpu_scache_line_size(); 313 unsigned long sc_lsize = cpu_scache_line_size();
314 314
@@ -326,7 +326,7 @@ static void __cpuinit r4k_blast_scache_page_indexed_setup(void)
326 326
327static void (* r4k_blast_scache)(void); 327static void (* r4k_blast_scache)(void);
328 328
329static void __cpuinit r4k_blast_scache_setup(void) 329static void r4k_blast_scache_setup(void)
330{ 330{
331 unsigned long sc_lsize = cpu_scache_line_size(); 331 unsigned long sc_lsize = cpu_scache_line_size();
332 332
@@ -797,11 +797,11 @@ static inline void alias_74k_erratum(struct cpuinfo_mips *c)
797 } 797 }
798} 798}
799 799
800static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way", 800static char *way_string[] = { NULL, "direct mapped", "2-way",
801 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 801 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
802}; 802};
803 803
804static void __cpuinit probe_pcache(void) 804static void probe_pcache(void)
805{ 805{
806 struct cpuinfo_mips *c = &current_cpu_data; 806 struct cpuinfo_mips *c = &current_cpu_data;
807 unsigned int config = read_c0_config(); 807 unsigned int config = read_c0_config();
@@ -1119,7 +1119,7 @@ static void __cpuinit probe_pcache(void)
1119 * executes in KSEG1 space or else you will crash and burn badly. You have 1119 * executes in KSEG1 space or else you will crash and burn badly. You have
1120 * been warned. 1120 * been warned.
1121 */ 1121 */
1122static int __cpuinit probe_scache(void) 1122static int probe_scache(void)
1123{ 1123{
1124 unsigned long flags, addr, begin, end, pow2; 1124 unsigned long flags, addr, begin, end, pow2;
1125 unsigned int config = read_c0_config(); 1125 unsigned int config = read_c0_config();
@@ -1196,7 +1196,7 @@ extern int r5k_sc_init(void);
1196extern int rm7k_sc_init(void); 1196extern int rm7k_sc_init(void);
1197extern int mips_sc_init(void); 1197extern int mips_sc_init(void);
1198 1198
1199static void __cpuinit setup_scache(void) 1199static void setup_scache(void)
1200{ 1200{
1201 struct cpuinfo_mips *c = &current_cpu_data; 1201 struct cpuinfo_mips *c = &current_cpu_data;
1202 unsigned int config = read_c0_config(); 1202 unsigned int config = read_c0_config();
@@ -1329,7 +1329,7 @@ static void nxp_pr4450_fixup_config(void)
1329 NXP_BARRIER(); 1329 NXP_BARRIER();
1330} 1330}
1331 1331
1332static int __cpuinitdata cca = -1; 1332static int cca = -1;
1333 1333
1334static int __init cca_setup(char *str) 1334static int __init cca_setup(char *str)
1335{ 1335{
@@ -1340,7 +1340,7 @@ static int __init cca_setup(char *str)
1340 1340
1341early_param("cca", cca_setup); 1341early_param("cca", cca_setup);
1342 1342
1343static void __cpuinit coherency_setup(void) 1343static void coherency_setup(void)
1344{ 1344{
1345 if (cca < 0 || cca > 7) 1345 if (cca < 0 || cca > 7)
1346 cca = read_c0_config() & CONF_CM_CMASK; 1346 cca = read_c0_config() & CONF_CM_CMASK;
@@ -1380,7 +1380,7 @@ static void __cpuinit coherency_setup(void)
1380 } 1380 }
1381} 1381}
1382 1382
1383static void __cpuinit r4k_cache_error_setup(void) 1383static void r4k_cache_error_setup(void)
1384{ 1384{
1385 extern char __weak except_vec2_generic; 1385 extern char __weak except_vec2_generic;
1386 extern char __weak except_vec2_sb1; 1386 extern char __weak except_vec2_sb1;
@@ -1398,7 +1398,7 @@ static void __cpuinit r4k_cache_error_setup(void)
1398 } 1398 }
1399} 1399}
1400 1400
1401void __cpuinit r4k_cache_init(void) 1401void r4k_cache_init(void)
1402{ 1402{
1403 extern void build_clear_page(void); 1403 extern void build_clear_page(void);
1404 extern void build_copy_page(void); 1404 extern void build_copy_page(void);
diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c
index ba9da270289f..8d909dbbf37f 100644
--- a/arch/mips/mm/c-tx39.c
+++ b/arch/mips/mm/c-tx39.c
@@ -344,7 +344,7 @@ static __init void tx39_probe_cache(void)
344 } 344 }
345} 345}
346 346
347void __cpuinit tx39_cache_init(void) 347void tx39_cache_init(void)
348{ 348{
349 extern void build_clear_page(void); 349 extern void build_clear_page(void);
350 extern void build_copy_page(void); 350 extern void build_copy_page(void);
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
index 5aeb3eb0b72f..15f813c303b4 100644
--- a/arch/mips/mm/cache.c
+++ b/arch/mips/mm/cache.c
@@ -182,7 +182,7 @@ static inline void setup_protection_map(void)
182 } 182 }
183} 183}
184 184
185void __cpuinit cpu_cache_init(void) 185void cpu_cache_init(void)
186{ 186{
187 if (cpu_has_3k_cache) { 187 if (cpu_has_3k_cache) {
188 extern void __weak r3k_cache_init(void); 188 extern void __weak r3k_cache_init(void);
diff --git a/arch/mips/mm/cerr-sb1.c b/arch/mips/mm/cerr-sb1.c
index 576add33bf5b..ee5c1ff861ae 100644
--- a/arch/mips/mm/cerr-sb1.c
+++ b/arch/mips/mm/cerr-sb1.c
@@ -182,11 +182,7 @@ asmlinkage void sb1_cache_error(void)
182 182
183#ifdef CONFIG_SIBYTE_BW_TRACE 183#ifdef CONFIG_SIBYTE_BW_TRACE
184 /* Freeze the trace buffer now */ 184 /* Freeze the trace buffer now */
185#if defined(CONFIG_SIBYTE_BCM1x55) || defined(CONFIG_SIBYTE_BCM1x80)
186 csr_out32(M_BCM1480_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG));
187#else
188 csr_out32(M_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG)); 185 csr_out32(M_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG));
189#endif
190 printk("Trace buffer frozen\n"); 186 printk("Trace buffer frozen\n");
191#endif 187#endif
192 188
diff --git a/arch/mips/mm/cex-sb1.S b/arch/mips/mm/cex-sb1.S
index fe1d887e8d70..191cf6e0c725 100644
--- a/arch/mips/mm/cex-sb1.S
+++ b/arch/mips/mm/cex-sb1.S
@@ -49,8 +49,6 @@
49 * (0x170-0x17f) are used to preserve k0, k1, and ra. 49 * (0x170-0x17f) are used to preserve k0, k1, and ra.
50 */ 50 */
51 51
52 __CPUINIT
53
54LEAF(except_vec2_sb1) 52LEAF(except_vec2_sb1)
55 /* 53 /*
56 * If this error is recoverable, we need to exit the handler 54 * If this error is recoverable, we need to exit the handler
@@ -142,8 +140,6 @@ unrecoverable:
142 140
143END(except_vec2_sb1) 141END(except_vec2_sb1)
144 142
145 __FINIT
146
147 LEAF(handle_vec2_sb1) 143 LEAF(handle_vec2_sb1)
148 mfc0 k0,CP0_CONFIG 144 mfc0 k0,CP0_CONFIG
149 li k1,~CONF_CM_CMASK 145 li k1,~CONF_CM_CMASK
diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c
index caf92ecb37d6..aaccf1c10699 100644
--- a/arch/mips/mm/dma-default.c
+++ b/arch/mips/mm/dma-default.c
@@ -246,6 +246,9 @@ static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
246 if (!plat_device_is_coherent(dev)) 246 if (!plat_device_is_coherent(dev))
247 __dma_sync(sg_page(sg), sg->offset, sg->length, 247 __dma_sync(sg_page(sg), sg->offset, sg->length,
248 direction); 248 direction);
249#ifdef CONFIG_NEED_SG_DMA_LENGTH
250 sg->dma_length = sg->length;
251#endif
249 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) + 252 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
250 sg->offset; 253 sg->offset;
251 } 254 }
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index 0fead53d1c26..85df1cd8d446 100644
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -5,6 +5,7 @@
5 * 5 *
6 * Copyright (C) 1995 - 2000 by Ralf Baechle 6 * Copyright (C) 1995 - 2000 by Ralf Baechle
7 */ 7 */
8#include <linux/context_tracking.h>
8#include <linux/signal.h> 9#include <linux/signal.h>
9#include <linux/sched.h> 10#include <linux/sched.h>
10#include <linux/interrupt.h> 11#include <linux/interrupt.h>
@@ -32,8 +33,8 @@
32 * and the problem, and then passes it off to one of the appropriate 33 * and the problem, and then passes it off to one of the appropriate
33 * routines. 34 * routines.
34 */ 35 */
35asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, unsigned long write, 36static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
36 unsigned long address) 37 unsigned long address)
37{ 38{
38 struct vm_area_struct * vma = NULL; 39 struct vm_area_struct * vma = NULL;
39 struct task_struct *tsk = current; 40 struct task_struct *tsk = current;
@@ -312,3 +313,13 @@ vmalloc_fault:
312 } 313 }
313#endif 314#endif
314} 315}
316
317asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
318 unsigned long write, unsigned long address)
319{
320 enum ctx_state prev_state;
321
322 prev_state = exception_enter();
323 __do_page_fault(regs, write, address);
324 exception_exit(prev_state);
325}
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 9b973e0af9cb..4e73f10a7519 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -359,11 +359,24 @@ void __init paging_init(void)
359static struct kcore_list kcore_kseg0; 359static struct kcore_list kcore_kseg0;
360#endif 360#endif
361 361
362void __init mem_init(void) 362static inline void mem_init_free_highmem(void)
363{ 363{
364 unsigned long codesize, reservedpages, datasize, initsize; 364#ifdef CONFIG_HIGHMEM
365 unsigned long tmp, ram; 365 unsigned long tmp;
366 366
367 for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
368 struct page *page = pfn_to_page(tmp);
369
370 if (!page_is_ram(tmp))
371 SetPageReserved(page);
372 else
373 free_highmem_page(page);
374 }
375#endif
376}
377
378void __init mem_init(void)
379{
367#ifdef CONFIG_HIGHMEM 380#ifdef CONFIG_HIGHMEM
368#ifdef CONFIG_DISCONTIGMEM 381#ifdef CONFIG_DISCONTIGMEM
369#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet" 382#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
@@ -374,34 +387,10 @@ void __init mem_init(void)
374#endif 387#endif
375 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); 388 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
376 389
377 totalram_pages += free_all_bootmem(); 390 free_all_bootmem();
378 setup_zero_pages(); /* Setup zeroed pages. */ 391 setup_zero_pages(); /* Setup zeroed pages. */
379 392 mem_init_free_highmem();
380 reservedpages = ram = 0; 393 mem_init_print_info(NULL);
381 for (tmp = 0; tmp < max_low_pfn; tmp++)
382 if (page_is_ram(tmp) && pfn_valid(tmp)) {
383 ram++;
384 if (PageReserved(pfn_to_page(tmp)))
385 reservedpages++;
386 }
387 num_physpages = ram;
388
389#ifdef CONFIG_HIGHMEM
390 for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
391 struct page *page = pfn_to_page(tmp);
392
393 if (!page_is_ram(tmp)) {
394 SetPageReserved(page);
395 continue;
396 }
397 free_highmem_page(page);
398 }
399 num_physpages += totalhigh_pages;
400#endif
401
402 codesize = (unsigned long) &_etext - (unsigned long) &_text;
403 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
404 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
405 394
406#ifdef CONFIG_64BIT 395#ifdef CONFIG_64BIT
407 if ((unsigned long) &_text > (unsigned long) CKSEG0) 396 if ((unsigned long) &_text > (unsigned long) CKSEG0)
@@ -410,16 +399,6 @@ void __init mem_init(void)
410 kclist_add(&kcore_kseg0, (void *) CKSEG0, 399 kclist_add(&kcore_kseg0, (void *) CKSEG0,
411 0x80000000 - 4, KCORE_TEXT); 400 0x80000000 - 4, KCORE_TEXT);
412#endif 401#endif
413
414 printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
415 "%ldk reserved, %ldk data, %ldk init, %ldk highmem)\n",
416 nr_free_pages() << (PAGE_SHIFT-10),
417 ram << (PAGE_SHIFT-10),
418 codesize >> 10,
419 reservedpages << (PAGE_SHIFT-10),
420 datasize >> 10,
421 initsize >> 10,
422 totalhigh_pages << (PAGE_SHIFT-10));
423} 402}
424#endif /* !CONFIG_NEED_MULTIPLE_NODES */ 403#endif /* !CONFIG_NEED_MULTIPLE_NODES */
425 404
@@ -440,7 +419,8 @@ void free_init_pages(const char *what, unsigned long begin, unsigned long end)
440#ifdef CONFIG_BLK_DEV_INITRD 419#ifdef CONFIG_BLK_DEV_INITRD
441void free_initrd_mem(unsigned long start, unsigned long end) 420void free_initrd_mem(unsigned long start, unsigned long end)
442{ 421{
443 free_reserved_area(start, end, POISON_FREE_INITMEM, "initrd"); 422 free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
423 "initrd");
444} 424}
445#endif 425#endif
446 426
diff --git a/arch/mips/mm/mmap.c b/arch/mips/mm/mmap.c
index 7e5fe2790d8a..f1baadd56e82 100644
--- a/arch/mips/mm/mmap.c
+++ b/arch/mips/mm/mmap.c
@@ -158,11 +158,9 @@ void arch_pick_mmap_layout(struct mm_struct *mm)
158 if (mmap_is_legacy()) { 158 if (mmap_is_legacy()) {
159 mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; 159 mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
160 mm->get_unmapped_area = arch_get_unmapped_area; 160 mm->get_unmapped_area = arch_get_unmapped_area;
161 mm->unmap_area = arch_unmap_area;
162 } else { 161 } else {
163 mm->mmap_base = mmap_base(random_factor); 162 mm->mmap_base = mmap_base(random_factor);
164 mm->get_unmapped_area = arch_get_unmapped_area_topdown; 163 mm->get_unmapped_area = arch_get_unmapped_area_topdown;
165 mm->unmap_area = arch_unmap_area_topdown;
166 } 164 }
167} 165}
168 166
diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c
index 4eb8dcfaf1ce..218c2109a55d 100644
--- a/arch/mips/mm/page.c
+++ b/arch/mips/mm/page.c
@@ -66,29 +66,29 @@ UASM_L_LA(_copy_pref_both)
66UASM_L_LA(_copy_pref_store) 66UASM_L_LA(_copy_pref_store)
67 67
68/* We need one branch and therefore one relocation per target label. */ 68/* We need one branch and therefore one relocation per target label. */
69static struct uasm_label __cpuinitdata labels[5]; 69static struct uasm_label labels[5];
70static struct uasm_reloc __cpuinitdata relocs[5]; 70static struct uasm_reloc relocs[5];
71 71
72#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 72#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
73#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 73#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
74 74
75static int pref_bias_clear_store __cpuinitdata; 75static int pref_bias_clear_store;
76static int pref_bias_copy_load __cpuinitdata; 76static int pref_bias_copy_load;
77static int pref_bias_copy_store __cpuinitdata; 77static int pref_bias_copy_store;
78 78
79static u32 pref_src_mode __cpuinitdata; 79static u32 pref_src_mode;
80static u32 pref_dst_mode __cpuinitdata; 80static u32 pref_dst_mode;
81 81
82static int clear_word_size __cpuinitdata; 82static int clear_word_size;
83static int copy_word_size __cpuinitdata; 83static int copy_word_size;
84 84
85static int half_clear_loop_size __cpuinitdata; 85static int half_clear_loop_size;
86static int half_copy_loop_size __cpuinitdata; 86static int half_copy_loop_size;
87 87
88static int cache_line_size __cpuinitdata; 88static int cache_line_size;
89#define cache_line_mask() (cache_line_size - 1) 89#define cache_line_mask() (cache_line_size - 1)
90 90
91static inline void __cpuinit 91static inline void
92pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) 92pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
93{ 93{
94 if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) { 94 if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) {
@@ -108,7 +108,7 @@ pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
108 } 108 }
109} 109}
110 110
111static void __cpuinit set_prefetch_parameters(void) 111static void set_prefetch_parameters(void)
112{ 112{
113 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) 113 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg)
114 clear_word_size = 8; 114 clear_word_size = 8;
@@ -199,7 +199,7 @@ static void __cpuinit set_prefetch_parameters(void)
199 4 * copy_word_size)); 199 4 * copy_word_size));
200} 200}
201 201
202static void __cpuinit build_clear_store(u32 **buf, int off) 202static void build_clear_store(u32 **buf, int off)
203{ 203{
204 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) { 204 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) {
205 uasm_i_sd(buf, ZERO, off, A0); 205 uasm_i_sd(buf, ZERO, off, A0);
@@ -208,7 +208,7 @@ static void __cpuinit build_clear_store(u32 **buf, int off)
208 } 208 }
209} 209}
210 210
211static inline void __cpuinit build_clear_pref(u32 **buf, int off) 211static inline void build_clear_pref(u32 **buf, int off)
212{ 212{
213 if (off & cache_line_mask()) 213 if (off & cache_line_mask())
214 return; 214 return;
@@ -232,7 +232,7 @@ static inline void __cpuinit build_clear_pref(u32 **buf, int off)
232 232
233 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); 233 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
234 } 234 }
235 } 235 }
236} 236}
237 237
238extern u32 __clear_page_start; 238extern u32 __clear_page_start;
@@ -240,7 +240,7 @@ extern u32 __clear_page_end;
240extern u32 __copy_page_start; 240extern u32 __copy_page_start;
241extern u32 __copy_page_end; 241extern u32 __copy_page_end;
242 242
243void __cpuinit build_clear_page(void) 243void build_clear_page(void)
244{ 244{
245 int off; 245 int off;
246 u32 *buf = &__clear_page_start; 246 u32 *buf = &__clear_page_start;
@@ -333,7 +333,7 @@ void __cpuinit build_clear_page(void)
333 pr_debug("\t.set pop\n"); 333 pr_debug("\t.set pop\n");
334} 334}
335 335
336static void __cpuinit build_copy_load(u32 **buf, int reg, int off) 336static void build_copy_load(u32 **buf, int reg, int off)
337{ 337{
338 if (cpu_has_64bit_gp_regs) { 338 if (cpu_has_64bit_gp_regs) {
339 uasm_i_ld(buf, reg, off, A1); 339 uasm_i_ld(buf, reg, off, A1);
@@ -342,7 +342,7 @@ static void __cpuinit build_copy_load(u32 **buf, int reg, int off)
342 } 342 }
343} 343}
344 344
345static void __cpuinit build_copy_store(u32 **buf, int reg, int off) 345static void build_copy_store(u32 **buf, int reg, int off)
346{ 346{
347 if (cpu_has_64bit_gp_regs) { 347 if (cpu_has_64bit_gp_regs) {
348 uasm_i_sd(buf, reg, off, A0); 348 uasm_i_sd(buf, reg, off, A0);
@@ -387,7 +387,7 @@ static inline void build_copy_store_pref(u32 **buf, int off)
387 } 387 }
388} 388}
389 389
390void __cpuinit build_copy_page(void) 390void build_copy_page(void)
391{ 391{
392 int off; 392 int off;
393 u32 *buf = &__copy_page_start; 393 u32 *buf = &__copy_page_start;
diff --git a/arch/mips/mm/sc-ip22.c b/arch/mips/mm/sc-ip22.c
index c6aaed934d53..dc7c5a5214a9 100644
--- a/arch/mips/mm/sc-ip22.c
+++ b/arch/mips/mm/sc-ip22.c
@@ -167,7 +167,7 @@ static struct bcache_ops indy_sc_ops = {
167 .bc_inv = indy_sc_wback_invalidate 167 .bc_inv = indy_sc_wback_invalidate
168}; 168};
169 169
170void __cpuinit indy_sc_init(void) 170void indy_sc_init(void)
171{ 171{
172 if (indy_sc_probe()) { 172 if (indy_sc_probe()) {
173 indy_sc_enable(); 173 indy_sc_enable();
diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
index df96da7e939b..5d01392e3518 100644
--- a/arch/mips/mm/sc-mips.c
+++ b/arch/mips/mm/sc-mips.c
@@ -132,7 +132,7 @@ static inline int __init mips_sc_probe(void)
132 return 1; 132 return 1;
133} 133}
134 134
135int __cpuinit mips_sc_init(void) 135int mips_sc_init(void)
136{ 136{
137 int found = mips_sc_probe(); 137 int found = mips_sc_probe();
138 if (found) { 138 if (found) {
diff --git a/arch/mips/mm/sc-r5k.c b/arch/mips/mm/sc-r5k.c
index 8bc67720e145..0216ed6eaa2a 100644
--- a/arch/mips/mm/sc-r5k.c
+++ b/arch/mips/mm/sc-r5k.c
@@ -98,7 +98,7 @@ static struct bcache_ops r5k_sc_ops = {
98 .bc_inv = r5k_dma_cache_inv_sc 98 .bc_inv = r5k_dma_cache_inv_sc
99}; 99};
100 100
101void __cpuinit r5k_sc_init(void) 101void r5k_sc_init(void)
102{ 102{
103 if (r5k_sc_probe()) { 103 if (r5k_sc_probe()) {
104 r5k_sc_enable(); 104 r5k_sc_enable();
diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c
index 274af3be1442..aaffbba33706 100644
--- a/arch/mips/mm/sc-rm7k.c
+++ b/arch/mips/mm/sc-rm7k.c
@@ -104,7 +104,7 @@ static void blast_rm7k_tcache(void)
104/* 104/*
105 * This function is executed in uncached address space. 105 * This function is executed in uncached address space.
106 */ 106 */
107static __cpuinit void __rm7k_tc_enable(void) 107static void __rm7k_tc_enable(void)
108{ 108{
109 int i; 109 int i;
110 110
@@ -117,7 +117,7 @@ static __cpuinit void __rm7k_tc_enable(void)
117 cache_op(Index_Store_Tag_T, CKSEG0ADDR(i)); 117 cache_op(Index_Store_Tag_T, CKSEG0ADDR(i));
118} 118}
119 119
120static __cpuinit void rm7k_tc_enable(void) 120static void rm7k_tc_enable(void)
121{ 121{
122 if (read_c0_config() & RM7K_CONF_TE) 122 if (read_c0_config() & RM7K_CONF_TE)
123 return; 123 return;
@@ -130,7 +130,7 @@ static __cpuinit void rm7k_tc_enable(void)
130/* 130/*
131 * This function is executed in uncached address space. 131 * This function is executed in uncached address space.
132 */ 132 */
133static __cpuinit void __rm7k_sc_enable(void) 133static void __rm7k_sc_enable(void)
134{ 134{
135 int i; 135 int i;
136 136
@@ -143,7 +143,7 @@ static __cpuinit void __rm7k_sc_enable(void)
143 cache_op(Index_Store_Tag_SD, CKSEG0ADDR(i)); 143 cache_op(Index_Store_Tag_SD, CKSEG0ADDR(i));
144} 144}
145 145
146static __cpuinit void rm7k_sc_enable(void) 146static void rm7k_sc_enable(void)
147{ 147{
148 if (read_c0_config() & RM7K_CONF_SE) 148 if (read_c0_config() & RM7K_CONF_SE)
149 return; 149 return;
@@ -184,7 +184,7 @@ static struct bcache_ops rm7k_sc_ops = {
184 * This is a probing function like the one found in c-r4k.c, we look for the 184 * This is a probing function like the one found in c-r4k.c, we look for the
185 * wrap around point with different addresses. 185 * wrap around point with different addresses.
186 */ 186 */
187static __cpuinit void __probe_tcache(void) 187static void __probe_tcache(void)
188{ 188{
189 unsigned long flags, addr, begin, end, pow2; 189 unsigned long flags, addr, begin, end, pow2;
190 190
@@ -226,7 +226,7 @@ static __cpuinit void __probe_tcache(void)
226 local_irq_restore(flags); 226 local_irq_restore(flags);
227} 227}
228 228
229void __cpuinit rm7k_sc_init(void) 229void rm7k_sc_init(void)
230{ 230{
231 struct cpuinfo_mips *c = &current_cpu_data; 231 struct cpuinfo_mips *c = &current_cpu_data;
232 unsigned int config = read_c0_config(); 232 unsigned int config = read_c0_config();
diff --git a/arch/mips/mm/tlb-funcs.S b/arch/mips/mm/tlb-funcs.S
new file mode 100644
index 000000000000..30a494db99c2
--- /dev/null
+++ b/arch/mips/mm/tlb-funcs.S
@@ -0,0 +1,37 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Micro-assembler generated tlb handler functions.
7 *
8 * Copyright (C) 2013 Broadcom Corporation.
9 *
10 * Based on mm/page-funcs.c
11 * Copyright (C) 2012 MIPS Technologies, Inc.
12 * Copyright (C) 2012 Ralf Baechle <ralf@linux-mips.org>
13 */
14#include <asm/asm.h>
15#include <asm/regdef.h>
16
17#define FASTPATH_SIZE 128
18
19LEAF(tlbmiss_handler_setup_pgd)
20 .space 16 * 4
21END(tlbmiss_handler_setup_pgd)
22EXPORT(tlbmiss_handler_setup_pgd_end)
23
24LEAF(handle_tlbm)
25 .space FASTPATH_SIZE * 4
26END(handle_tlbm)
27EXPORT(handle_tlbm_end)
28
29LEAF(handle_tlbs)
30 .space FASTPATH_SIZE * 4
31END(handle_tlbs)
32EXPORT(handle_tlbs_end)
33
34LEAF(handle_tlbl)
35 .space FASTPATH_SIZE * 4
36END(handle_tlbl)
37EXPORT(handle_tlbl_end)
diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c
index a63d1ed0827f..9aca10994cd2 100644
--- a/arch/mips/mm/tlb-r3k.c
+++ b/arch/mips/mm/tlb-r3k.c
@@ -276,7 +276,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
276 } 276 }
277} 277}
278 278
279void __cpuinit tlb_init(void) 279void tlb_init(void)
280{ 280{
281 local_flush_tlb_all(); 281 local_flush_tlb_all();
282 282
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index c643de4c473a..00b26a67a06d 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -389,7 +389,7 @@ int __init has_transparent_hugepage(void)
389 389
390#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 390#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
391 391
392static int __cpuinitdata ntlb; 392static int ntlb;
393static int __init set_ntlb(char *str) 393static int __init set_ntlb(char *str)
394{ 394{
395 get_option(&str, &ntlb); 395 get_option(&str, &ntlb);
@@ -398,7 +398,7 @@ static int __init set_ntlb(char *str)
398 398
399__setup("ntlb=", set_ntlb); 399__setup("ntlb=", set_ntlb);
400 400
401void __cpuinit tlb_init(void) 401void tlb_init(void)
402{ 402{
403 /* 403 /*
404 * You should never change this register: 404 * You should never change this register:
diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c
index 91c2499f806a..6a99733a4440 100644
--- a/arch/mips/mm/tlb-r8k.c
+++ b/arch/mips/mm/tlb-r8k.c
@@ -213,14 +213,14 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
213 local_irq_restore(flags); 213 local_irq_restore(flags);
214} 214}
215 215
216static void __cpuinit probe_tlb(unsigned long config) 216static void probe_tlb(unsigned long config)
217{ 217{
218 struct cpuinfo_mips *c = &current_cpu_data; 218 struct cpuinfo_mips *c = &current_cpu_data;
219 219
220 c->tlbsize = 3 * 128; /* 3 sets each 128 entries */ 220 c->tlbsize = 3 * 128; /* 3 sets each 128 entries */
221} 221}
222 222
223void __cpuinit tlb_init(void) 223void tlb_init(void)
224{ 224{
225 unsigned int config = read_c0_config(); 225 unsigned int config = read_c0_config();
226 unsigned long status; 226 unsigned long status;
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index afeef93f81a7..556cb4815770 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -136,7 +136,7 @@ static int scratchpad_offset(int i)
136 * why; it's not an issue caused by the core RTL. 136 * why; it's not an issue caused by the core RTL.
137 * 137 *
138 */ 138 */
139static int __cpuinit m4kc_tlbp_war(void) 139static int m4kc_tlbp_war(void)
140{ 140{
141 return (current_cpu_data.processor_id & 0xffff00) == 141 return (current_cpu_data.processor_id & 0xffff00) ==
142 (PRID_COMP_MIPS | PRID_IMP_4KC); 142 (PRID_COMP_MIPS | PRID_IMP_4KC);
@@ -181,11 +181,9 @@ UASM_L_LA(_large_segbits_fault)
181UASM_L_LA(_tlb_huge_update) 181UASM_L_LA(_tlb_huge_update)
182#endif 182#endif
183 183
184static int __cpuinitdata hazard_instance; 184static int hazard_instance;
185 185
186static void __cpuinit uasm_bgezl_hazard(u32 **p, 186static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
187 struct uasm_reloc **r,
188 int instance)
189{ 187{
190 switch (instance) { 188 switch (instance) {
191 case 0 ... 7: 189 case 0 ... 7:
@@ -196,9 +194,7 @@ static void __cpuinit uasm_bgezl_hazard(u32 **p,
196 } 194 }
197} 195}
198 196
199static void __cpuinit uasm_bgezl_label(struct uasm_label **l, 197static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
200 u32 **p,
201 int instance)
202{ 198{
203 switch (instance) { 199 switch (instance) {
204 case 0 ... 7: 200 case 0 ... 7:
@@ -295,17 +291,28 @@ static inline void dump_handler(const char *symbol, const u32 *handler, int coun
295 * We deliberately chose a buffer size of 128, so we won't scribble 291 * We deliberately chose a buffer size of 128, so we won't scribble
296 * over anything important on overflow before we panic. 292 * over anything important on overflow before we panic.
297 */ 293 */
298static u32 tlb_handler[128] __cpuinitdata; 294static u32 tlb_handler[128];
299 295
300/* simply assume worst case size for labels and relocs */ 296/* simply assume worst case size for labels and relocs */
301static struct uasm_label labels[128] __cpuinitdata; 297static struct uasm_label labels[128];
302static struct uasm_reloc relocs[128] __cpuinitdata; 298static struct uasm_reloc relocs[128];
303 299
304static int check_for_high_segbits __cpuinitdata; 300static int check_for_high_segbits;
305 301
306static unsigned int kscratch_used_mask __cpuinitdata; 302static unsigned int kscratch_used_mask;
307 303
308static int __cpuinit allocate_kscratch(void) 304static inline int __maybe_unused c0_kscratch(void)
305{
306 switch (current_cpu_type()) {
307 case CPU_XLP:
308 case CPU_XLR:
309 return 22;
310 default:
311 return 31;
312 }
313}
314
315static int allocate_kscratch(void)
309{ 316{
310 int r; 317 int r;
311 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 318 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
@@ -322,11 +329,11 @@ static int __cpuinit allocate_kscratch(void)
322 return r; 329 return r;
323} 330}
324 331
325static int scratch_reg __cpuinitdata; 332static int scratch_reg;
326static int pgd_reg __cpuinitdata; 333static int pgd_reg;
327enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 334enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
328 335
329static struct work_registers __cpuinit build_get_work_registers(u32 **p) 336static struct work_registers build_get_work_registers(u32 **p)
330{ 337{
331 struct work_registers r; 338 struct work_registers r;
332 339
@@ -334,9 +341,9 @@ static struct work_registers __cpuinit build_get_work_registers(u32 **p)
334 int smp_processor_id_sel; 341 int smp_processor_id_sel;
335 int smp_processor_id_shift; 342 int smp_processor_id_shift;
336 343
337 if (scratch_reg > 0) { 344 if (scratch_reg >= 0) {
338 /* Save in CPU local C0_KScratch? */ 345 /* Save in CPU local C0_KScratch? */
339 UASM_i_MTC0(p, 1, 31, scratch_reg); 346 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
340 r.r1 = K0; 347 r.r1 = K0;
341 r.r2 = K1; 348 r.r2 = K1;
342 r.r3 = 1; 349 r.r3 = 1;
@@ -382,10 +389,10 @@ static struct work_registers __cpuinit build_get_work_registers(u32 **p)
382 return r; 389 return r;
383} 390}
384 391
385static void __cpuinit build_restore_work_registers(u32 **p) 392static void build_restore_work_registers(u32 **p)
386{ 393{
387 if (scratch_reg > 0) { 394 if (scratch_reg >= 0) {
388 UASM_i_MFC0(p, 1, 31, scratch_reg); 395 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
389 return; 396 return;
390 } 397 }
391 /* K0 already points to save area, restore $1 and $2 */ 398 /* K0 already points to save area, restore $1 and $2 */
@@ -407,7 +414,7 @@ extern unsigned long pgd_current[];
407/* 414/*
408 * The R3000 TLB handler is simple. 415 * The R3000 TLB handler is simple.
409 */ 416 */
410static void __cpuinit build_r3000_tlb_refill_handler(void) 417static void build_r3000_tlb_refill_handler(void)
411{ 418{
412 long pgdc = (long)pgd_current; 419 long pgdc = (long)pgd_current;
413 u32 *p; 420 u32 *p;
@@ -452,7 +459,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
452 * other one.To keep things simple, we first assume linear space, 459 * other one.To keep things simple, we first assume linear space,
453 * then we relocate it to the final handler layout as needed. 460 * then we relocate it to the final handler layout as needed.
454 */ 461 */
455static u32 final_handler[64] __cpuinitdata; 462static u32 final_handler[64];
456 463
457/* 464/*
458 * Hazards 465 * Hazards
@@ -476,7 +483,7 @@ static u32 final_handler[64] __cpuinitdata;
476 * 483 *
477 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 484 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
478 */ 485 */
479static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 486static void __maybe_unused build_tlb_probe_entry(u32 **p)
480{ 487{
481 switch (current_cpu_type()) { 488 switch (current_cpu_type()) {
482 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 489 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */
@@ -500,9 +507,9 @@ static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
500 */ 507 */
501enum tlb_write_entry { tlb_random, tlb_indexed }; 508enum tlb_write_entry { tlb_random, tlb_indexed };
502 509
503static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 510static void build_tlb_write_entry(u32 **p, struct uasm_label **l,
504 struct uasm_reloc **r, 511 struct uasm_reloc **r,
505 enum tlb_write_entry wmode) 512 enum tlb_write_entry wmode)
506{ 513{
507 void(*tlbw)(u32 **) = NULL; 514 void(*tlbw)(u32 **) = NULL;
508 515
@@ -636,8 +643,8 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
636 } 643 }
637} 644}
638 645
639static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 646static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
640 unsigned int reg) 647 unsigned int reg)
641{ 648{
642 if (cpu_has_rixi) { 649 if (cpu_has_rixi) {
643 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); 650 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
@@ -652,11 +659,9 @@ static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
652 659
653#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 660#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
654 661
655static __cpuinit void build_restore_pagemask(u32 **p, 662static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
656 struct uasm_reloc **r, 663 unsigned int tmp, enum label_id lid,
657 unsigned int tmp, 664 int restore_scratch)
658 enum label_id lid,
659 int restore_scratch)
660{ 665{
661 if (restore_scratch) { 666 if (restore_scratch) {
662 /* Reset default page size */ 667 /* Reset default page size */
@@ -673,8 +678,8 @@ static __cpuinit void build_restore_pagemask(u32 **p,
673 uasm_i_mtc0(p, 0, C0_PAGEMASK); 678 uasm_i_mtc0(p, 0, C0_PAGEMASK);
674 uasm_il_b(p, r, lid); 679 uasm_il_b(p, r, lid);
675 } 680 }
676 if (scratch_reg > 0) 681 if (scratch_reg >= 0)
677 UASM_i_MFC0(p, 1, 31, scratch_reg); 682 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
678 else 683 else
679 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 684 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
680 } else { 685 } else {
@@ -695,12 +700,11 @@ static __cpuinit void build_restore_pagemask(u32 **p,
695 } 700 }
696} 701}
697 702
698static __cpuinit void build_huge_tlb_write_entry(u32 **p, 703static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l,
699 struct uasm_label **l, 704 struct uasm_reloc **r,
700 struct uasm_reloc **r, 705 unsigned int tmp,
701 unsigned int tmp, 706 enum tlb_write_entry wmode,
702 enum tlb_write_entry wmode, 707 int restore_scratch)
703 int restore_scratch)
704{ 708{
705 /* Set huge page tlb entry size */ 709 /* Set huge page tlb entry size */
706 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 710 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
@@ -715,9 +719,9 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p,
715/* 719/*
716 * Check if Huge PTE is present, if so then jump to LABEL. 720 * Check if Huge PTE is present, if so then jump to LABEL.
717 */ 721 */
718static void __cpuinit 722static void
719build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 723build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
720 unsigned int pmd, int lid) 724 unsigned int pmd, int lid)
721{ 725{
722 UASM_i_LW(p, tmp, 0, pmd); 726 UASM_i_LW(p, tmp, 0, pmd);
723 if (use_bbit_insns()) { 727 if (use_bbit_insns()) {
@@ -728,9 +732,8 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
728 } 732 }
729} 733}
730 734
731static __cpuinit void build_huge_update_entries(u32 **p, 735static void build_huge_update_entries(u32 **p, unsigned int pte,
732 unsigned int pte, 736 unsigned int tmp)
733 unsigned int tmp)
734{ 737{
735 int small_sequence; 738 int small_sequence;
736 739
@@ -760,11 +763,10 @@ static __cpuinit void build_huge_update_entries(u32 **p,
760 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 763 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
761} 764}
762 765
763static __cpuinit void build_huge_handler_tail(u32 **p, 766static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r,
764 struct uasm_reloc **r, 767 struct uasm_label **l,
765 struct uasm_label **l, 768 unsigned int pte,
766 unsigned int pte, 769 unsigned int ptr)
767 unsigned int ptr)
768{ 770{
769#ifdef CONFIG_SMP 771#ifdef CONFIG_SMP
770 UASM_i_SC(p, pte, 0, ptr); 772 UASM_i_SC(p, pte, 0, ptr);
@@ -783,7 +785,7 @@ static __cpuinit void build_huge_handler_tail(u32 **p,
783 * TMP and PTR are scratch. 785 * TMP and PTR are scratch.
784 * TMP will be clobbered, PTR will hold the pmd entry. 786 * TMP will be clobbered, PTR will hold the pmd entry.
785 */ 787 */
786static void __cpuinit 788static void
787build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 789build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
788 unsigned int tmp, unsigned int ptr) 790 unsigned int tmp, unsigned int ptr)
789{ 791{
@@ -817,7 +819,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
817#ifdef CONFIG_MIPS_PGD_C0_CONTEXT 819#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
818 if (pgd_reg != -1) { 820 if (pgd_reg != -1) {
819 /* pgd is in pgd_reg */ 821 /* pgd is in pgd_reg */
820 UASM_i_MFC0(p, ptr, 31, pgd_reg); 822 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
821 } else { 823 } else {
822 /* 824 /*
823 * &pgd << 11 stored in CONTEXT [23..63]. 825 * &pgd << 11 stored in CONTEXT [23..63].
@@ -875,7 +877,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
875 * BVADDR is the faulting address, PTR is scratch. 877 * BVADDR is the faulting address, PTR is scratch.
876 * PTR will hold the pgd for vmalloc. 878 * PTR will hold the pgd for vmalloc.
877 */ 879 */
878static void __cpuinit 880static void
879build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 881build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
880 unsigned int bvaddr, unsigned int ptr, 882 unsigned int bvaddr, unsigned int ptr,
881 enum vmalloc64_mode mode) 883 enum vmalloc64_mode mode)
@@ -929,8 +931,8 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
929 uasm_i_jr(p, ptr); 931 uasm_i_jr(p, ptr);
930 932
931 if (mode == refill_scratch) { 933 if (mode == refill_scratch) {
932 if (scratch_reg > 0) 934 if (scratch_reg >= 0)
933 UASM_i_MFC0(p, 1, 31, scratch_reg); 935 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
934 else 936 else
935 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 937 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
936 } else { 938 } else {
@@ -945,7 +947,7 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
945 * TMP and PTR are scratch. 947 * TMP and PTR are scratch.
946 * TMP will be clobbered, PTR will hold the pgd entry. 948 * TMP will be clobbered, PTR will hold the pgd entry.
947 */ 949 */
948static void __cpuinit __maybe_unused 950static void __maybe_unused
949build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 951build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
950{ 952{
951 long pgdc = (long)pgd_current; 953 long pgdc = (long)pgd_current;
@@ -961,7 +963,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
961 uasm_i_srl(p, ptr, ptr, 19); 963 uasm_i_srl(p, ptr, ptr, 19);
962#else 964#else
963 /* 965 /*
964 * smp_processor_id() << 3 is stored in CONTEXT. 966 * smp_processor_id() << 2 is stored in CONTEXT.
965 */ 967 */
966 uasm_i_mfc0(p, ptr, C0_CONTEXT); 968 uasm_i_mfc0(p, ptr, C0_CONTEXT);
967 UASM_i_LA_mostly(p, tmp, pgdc); 969 UASM_i_LA_mostly(p, tmp, pgdc);
@@ -980,7 +982,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
980 982
981#endif /* !CONFIG_64BIT */ 983#endif /* !CONFIG_64BIT */
982 984
983static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 985static void build_adjust_context(u32 **p, unsigned int ctx)
984{ 986{
985 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 987 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
986 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 988 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
@@ -1006,7 +1008,7 @@ static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
1006 uasm_i_andi(p, ctx, ctx, mask); 1008 uasm_i_andi(p, ctx, ctx, mask);
1007} 1009}
1008 1010
1009static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 1011static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
1010{ 1012{
1011 /* 1013 /*
1012 * Bug workaround for the Nevada. It seems as if under certain 1014 * Bug workaround for the Nevada. It seems as if under certain
@@ -1031,8 +1033,7 @@ static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr
1031 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 1033 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
1032} 1034}
1033 1035
1034static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 1036static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep)
1035 unsigned int ptep)
1036{ 1037{
1037 /* 1038 /*
1038 * 64bit address support (36bit on a 32bit CPU) in a 32bit 1039 * 64bit address support (36bit on a 32bit CPU) in a 32bit
@@ -1093,10 +1094,10 @@ struct mips_huge_tlb_info {
1093 int restore_scratch; 1094 int restore_scratch;
1094}; 1095};
1095 1096
1096static struct mips_huge_tlb_info __cpuinit 1097static struct mips_huge_tlb_info
1097build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1098build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1098 struct uasm_reloc **r, unsigned int tmp, 1099 struct uasm_reloc **r, unsigned int tmp,
1099 unsigned int ptr, int c0_scratch) 1100 unsigned int ptr, int c0_scratch_reg)
1100{ 1101{
1101 struct mips_huge_tlb_info rv; 1102 struct mips_huge_tlb_info rv;
1102 unsigned int even, odd; 1103 unsigned int even, odd;
@@ -1110,12 +1111,12 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1110 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1111 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1111 1112
1112 if (pgd_reg != -1) 1113 if (pgd_reg != -1)
1113 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1114 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
1114 else 1115 else
1115 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1116 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1116 1117
1117 if (c0_scratch >= 0) 1118 if (c0_scratch_reg >= 0)
1118 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1119 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1119 else 1120 else
1120 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1121 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1121 1122
@@ -1130,14 +1131,14 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1130 } 1131 }
1131 } else { 1132 } else {
1132 if (pgd_reg != -1) 1133 if (pgd_reg != -1)
1133 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1134 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
1134 else 1135 else
1135 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1136 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1136 1137
1137 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1138 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1138 1139
1139 if (c0_scratch >= 0) 1140 if (c0_scratch_reg >= 0)
1140 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1141 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1141 else 1142 else
1142 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1143 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1143 1144
@@ -1242,8 +1243,8 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1242 } 1243 }
1243 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1244 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
1244 1245
1245 if (c0_scratch >= 0) { 1246 if (c0_scratch_reg >= 0) {
1246 UASM_i_MFC0(p, scratch, 31, c0_scratch); 1247 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1247 build_tlb_write_entry(p, l, r, tlb_random); 1248 build_tlb_write_entry(p, l, r, tlb_random);
1248 uasm_l_leave(l, *p); 1249 uasm_l_leave(l, *p);
1249 rv.restore_scratch = 1; 1250 rv.restore_scratch = 1;
@@ -1271,7 +1272,7 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1271 */ 1272 */
1272#define MIPS64_REFILL_INSNS 32 1273#define MIPS64_REFILL_INSNS 32
1273 1274
1274static void __cpuinit build_r4000_tlb_refill_handler(void) 1275static void build_r4000_tlb_refill_handler(void)
1275{ 1276{
1276 u32 *p = tlb_handler; 1277 u32 *p = tlb_handler;
1277 struct uasm_label *l = labels; 1278 struct uasm_label *l = labels;
@@ -1286,7 +1287,7 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
1286 memset(relocs, 0, sizeof(relocs)); 1287 memset(relocs, 0, sizeof(relocs));
1287 memset(final_handler, 0, sizeof(final_handler)); 1288 memset(final_handler, 0, sizeof(final_handler));
1288 1289
1289 if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) { 1290 if ((scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
1290 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1291 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
1291 scratch_reg); 1292 scratch_reg);
1292 vmalloc_mode = refill_scratch; 1293 vmalloc_mode = refill_scratch;
@@ -1444,27 +1445,25 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
1444 dump_handler("r4000_tlb_refill", (u32 *)ebase, 64); 1445 dump_handler("r4000_tlb_refill", (u32 *)ebase, 64);
1445} 1446}
1446 1447
1447/* 1448extern u32 handle_tlbl[], handle_tlbl_end[];
1448 * 128 instructions for the fastpath handler is generous and should 1449extern u32 handle_tlbs[], handle_tlbs_end[];
1449 * never be exceeded. 1450extern u32 handle_tlbm[], handle_tlbm_end[];
1450 */
1451#define FASTPATH_SIZE 128
1452 1451
1453u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
1454u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
1455u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
1456#ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1452#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
1457u32 tlbmiss_handler_setup_pgd_array[16] __cacheline_aligned; 1453extern u32 tlbmiss_handler_setup_pgd[], tlbmiss_handler_setup_pgd_end[];
1458 1454
1459static void __cpuinit build_r4000_setup_pgd(void) 1455static void build_r4000_setup_pgd(void)
1460{ 1456{
1461 const int a0 = 4; 1457 const int a0 = 4;
1462 const int a1 = 5; 1458 const int a1 = 5;
1463 u32 *p = tlbmiss_handler_setup_pgd_array; 1459 u32 *p = tlbmiss_handler_setup_pgd;
1460 const int tlbmiss_handler_setup_pgd_size =
1461 tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd;
1464 struct uasm_label *l = labels; 1462 struct uasm_label *l = labels;
1465 struct uasm_reloc *r = relocs; 1463 struct uasm_reloc *r = relocs;
1466 1464
1467 memset(tlbmiss_handler_setup_pgd_array, 0, sizeof(tlbmiss_handler_setup_pgd_array)); 1465 memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
1466 sizeof(tlbmiss_handler_setup_pgd[0]));
1468 memset(labels, 0, sizeof(labels)); 1467 memset(labels, 0, sizeof(labels));
1469 memset(relocs, 0, sizeof(relocs)); 1468 memset(relocs, 0, sizeof(relocs));
1470 1469
@@ -1490,21 +1489,21 @@ static void __cpuinit build_r4000_setup_pgd(void)
1490 } else { 1489 } else {
1491 /* PGD in c0_KScratch */ 1490 /* PGD in c0_KScratch */
1492 uasm_i_jr(&p, 31); 1491 uasm_i_jr(&p, 31);
1493 UASM_i_MTC0(&p, a0, 31, pgd_reg); 1492 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
1494 } 1493 }
1495 if (p - tlbmiss_handler_setup_pgd_array > ARRAY_SIZE(tlbmiss_handler_setup_pgd_array)) 1494 if (p >= tlbmiss_handler_setup_pgd_end)
1496 panic("tlbmiss_handler_setup_pgd_array space exceeded"); 1495 panic("tlbmiss_handler_setup_pgd space exceeded");
1496
1497 uasm_resolve_relocs(relocs, labels); 1497 uasm_resolve_relocs(relocs, labels);
1498 pr_debug("Wrote tlbmiss_handler_setup_pgd_array (%u instructions).\n", 1498 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
1499 (unsigned int)(p - tlbmiss_handler_setup_pgd_array)); 1499 (unsigned int)(p - tlbmiss_handler_setup_pgd));
1500 1500
1501 dump_handler("tlbmiss_handler", 1501 dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
1502 tlbmiss_handler_setup_pgd_array, 1502 tlbmiss_handler_setup_pgd_size);
1503 ARRAY_SIZE(tlbmiss_handler_setup_pgd_array));
1504} 1503}
1505#endif 1504#endif
1506 1505
1507static void __cpuinit 1506static void
1508iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1507iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
1509{ 1508{
1510#ifdef CONFIG_SMP 1509#ifdef CONFIG_SMP
@@ -1524,7 +1523,7 @@ iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
1524#endif 1523#endif
1525} 1524}
1526 1525
1527static void __cpuinit 1526static void
1528iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1527iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
1529 unsigned int mode) 1528 unsigned int mode)
1530{ 1529{
@@ -1584,7 +1583,7 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
1584 * the page table where this PTE is located, PTE will be re-loaded 1583 * the page table where this PTE is located, PTE will be re-loaded
1585 * with it's original value. 1584 * with it's original value.
1586 */ 1585 */
1587static void __cpuinit 1586static void
1588build_pte_present(u32 **p, struct uasm_reloc **r, 1587build_pte_present(u32 **p, struct uasm_reloc **r,
1589 int pte, int ptr, int scratch, enum label_id lid) 1588 int pte, int ptr, int scratch, enum label_id lid)
1590{ 1589{
@@ -1612,7 +1611,7 @@ build_pte_present(u32 **p, struct uasm_reloc **r,
1612} 1611}
1613 1612
1614/* Make PTE valid, store result in PTR. */ 1613/* Make PTE valid, store result in PTR. */
1615static void __cpuinit 1614static void
1616build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1615build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
1617 unsigned int ptr) 1616 unsigned int ptr)
1618{ 1617{
@@ -1625,7 +1624,7 @@ build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
1625 * Check if PTE can be written to, if not branch to LABEL. Regardless 1624 * Check if PTE can be written to, if not branch to LABEL. Regardless
1626 * restore PTE with value from PTR when done. 1625 * restore PTE with value from PTR when done.
1627 */ 1626 */
1628static void __cpuinit 1627static void
1629build_pte_writable(u32 **p, struct uasm_reloc **r, 1628build_pte_writable(u32 **p, struct uasm_reloc **r,
1630 unsigned int pte, unsigned int ptr, int scratch, 1629 unsigned int pte, unsigned int ptr, int scratch,
1631 enum label_id lid) 1630 enum label_id lid)
@@ -1645,7 +1644,7 @@ build_pte_writable(u32 **p, struct uasm_reloc **r,
1645/* Make PTE writable, update software status bits as well, then store 1644/* Make PTE writable, update software status bits as well, then store
1646 * at PTR. 1645 * at PTR.
1647 */ 1646 */
1648static void __cpuinit 1647static void
1649build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1648build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
1650 unsigned int ptr) 1649 unsigned int ptr)
1651{ 1650{
@@ -1659,7 +1658,7 @@ build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
1659 * Check if PTE can be modified, if not branch to LABEL. Regardless 1658 * Check if PTE can be modified, if not branch to LABEL. Regardless
1660 * restore PTE with value from PTR when done. 1659 * restore PTE with value from PTR when done.
1661 */ 1660 */
1662static void __cpuinit 1661static void
1663build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1662build_pte_modifiable(u32 **p, struct uasm_reloc **r,
1664 unsigned int pte, unsigned int ptr, int scratch, 1663 unsigned int pte, unsigned int ptr, int scratch,
1665 enum label_id lid) 1664 enum label_id lid)
@@ -1688,7 +1687,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
1688 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1687 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1689 * Then it returns. 1688 * Then it returns.
1690 */ 1689 */
1691static void __cpuinit 1690static void
1692build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1691build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1693{ 1692{
1694 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1693 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
@@ -1704,7 +1703,7 @@ build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1704 * may have the probe fail bit set as a result of a trap on a 1703 * may have the probe fail bit set as a result of a trap on a
1705 * kseg2 access, i.e. without refill. Then it returns. 1704 * kseg2 access, i.e. without refill. Then it returns.
1706 */ 1705 */
1707static void __cpuinit 1706static void
1708build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1707build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
1709 struct uasm_reloc **r, unsigned int pte, 1708 struct uasm_reloc **r, unsigned int pte,
1710 unsigned int tmp) 1709 unsigned int tmp)
@@ -1722,7 +1721,7 @@ build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
1722 uasm_i_rfe(p); /* branch delay */ 1721 uasm_i_rfe(p); /* branch delay */
1723} 1722}
1724 1723
1725static void __cpuinit 1724static void
1726build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1725build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1727 unsigned int ptr) 1726 unsigned int ptr)
1728{ 1727{
@@ -1742,13 +1741,14 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1742 uasm_i_tlbp(p); /* load delay */ 1741 uasm_i_tlbp(p); /* load delay */
1743} 1742}
1744 1743
1745static void __cpuinit build_r3000_tlb_load_handler(void) 1744static void build_r3000_tlb_load_handler(void)
1746{ 1745{
1747 u32 *p = handle_tlbl; 1746 u32 *p = handle_tlbl;
1747 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
1748 struct uasm_label *l = labels; 1748 struct uasm_label *l = labels;
1749 struct uasm_reloc *r = relocs; 1749 struct uasm_reloc *r = relocs;
1750 1750
1751 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1751 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
1752 memset(labels, 0, sizeof(labels)); 1752 memset(labels, 0, sizeof(labels));
1753 memset(relocs, 0, sizeof(relocs)); 1753 memset(relocs, 0, sizeof(relocs));
1754 1754
@@ -1762,23 +1762,24 @@ static void __cpuinit build_r3000_tlb_load_handler(void)
1762 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1762 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1763 uasm_i_nop(&p); 1763 uasm_i_nop(&p);
1764 1764
1765 if ((p - handle_tlbl) > FASTPATH_SIZE) 1765 if (p >= handle_tlbl_end)
1766 panic("TLB load handler fastpath space exceeded"); 1766 panic("TLB load handler fastpath space exceeded");
1767 1767
1768 uasm_resolve_relocs(relocs, labels); 1768 uasm_resolve_relocs(relocs, labels);
1769 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1769 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1770 (unsigned int)(p - handle_tlbl)); 1770 (unsigned int)(p - handle_tlbl));
1771 1771
1772 dump_handler("r3000_tlb_load", handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1772 dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size);
1773} 1773}
1774 1774
1775static void __cpuinit build_r3000_tlb_store_handler(void) 1775static void build_r3000_tlb_store_handler(void)
1776{ 1776{
1777 u32 *p = handle_tlbs; 1777 u32 *p = handle_tlbs;
1778 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
1778 struct uasm_label *l = labels; 1779 struct uasm_label *l = labels;
1779 struct uasm_reloc *r = relocs; 1780 struct uasm_reloc *r = relocs;
1780 1781
1781 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1782 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
1782 memset(labels, 0, sizeof(labels)); 1783 memset(labels, 0, sizeof(labels));
1783 memset(relocs, 0, sizeof(relocs)); 1784 memset(relocs, 0, sizeof(relocs));
1784 1785
@@ -1792,23 +1793,24 @@ static void __cpuinit build_r3000_tlb_store_handler(void)
1792 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1793 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1793 uasm_i_nop(&p); 1794 uasm_i_nop(&p);
1794 1795
1795 if ((p - handle_tlbs) > FASTPATH_SIZE) 1796 if (p >= handle_tlbs_end)
1796 panic("TLB store handler fastpath space exceeded"); 1797 panic("TLB store handler fastpath space exceeded");
1797 1798
1798 uasm_resolve_relocs(relocs, labels); 1799 uasm_resolve_relocs(relocs, labels);
1799 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1800 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1800 (unsigned int)(p - handle_tlbs)); 1801 (unsigned int)(p - handle_tlbs));
1801 1802
1802 dump_handler("r3000_tlb_store", handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1803 dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size);
1803} 1804}
1804 1805
1805static void __cpuinit build_r3000_tlb_modify_handler(void) 1806static void build_r3000_tlb_modify_handler(void)
1806{ 1807{
1807 u32 *p = handle_tlbm; 1808 u32 *p = handle_tlbm;
1809 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
1808 struct uasm_label *l = labels; 1810 struct uasm_label *l = labels;
1809 struct uasm_reloc *r = relocs; 1811 struct uasm_reloc *r = relocs;
1810 1812
1811 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1813 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
1812 memset(labels, 0, sizeof(labels)); 1814 memset(labels, 0, sizeof(labels));
1813 memset(relocs, 0, sizeof(relocs)); 1815 memset(relocs, 0, sizeof(relocs));
1814 1816
@@ -1822,21 +1824,21 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
1822 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1824 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1823 uasm_i_nop(&p); 1825 uasm_i_nop(&p);
1824 1826
1825 if ((p - handle_tlbm) > FASTPATH_SIZE) 1827 if (p >= handle_tlbm_end)
1826 panic("TLB modify handler fastpath space exceeded"); 1828 panic("TLB modify handler fastpath space exceeded");
1827 1829
1828 uasm_resolve_relocs(relocs, labels); 1830 uasm_resolve_relocs(relocs, labels);
1829 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1831 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1830 (unsigned int)(p - handle_tlbm)); 1832 (unsigned int)(p - handle_tlbm));
1831 1833
1832 dump_handler("r3000_tlb_modify", handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1834 dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size);
1833} 1835}
1834#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1836#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
1835 1837
1836/* 1838/*
1837 * R4000 style TLB load/store/modify handlers. 1839 * R4000 style TLB load/store/modify handlers.
1838 */ 1840 */
1839static struct work_registers __cpuinit 1841static struct work_registers
1840build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1842build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
1841 struct uasm_reloc **r) 1843 struct uasm_reloc **r)
1842{ 1844{
@@ -1872,7 +1874,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
1872 return wr; 1874 return wr;
1873} 1875}
1874 1876
1875static void __cpuinit 1877static void
1876build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1878build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
1877 struct uasm_reloc **r, unsigned int tmp, 1879 struct uasm_reloc **r, unsigned int tmp,
1878 unsigned int ptr) 1880 unsigned int ptr)
@@ -1890,14 +1892,15 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
1890#endif 1892#endif
1891} 1893}
1892 1894
1893static void __cpuinit build_r4000_tlb_load_handler(void) 1895static void build_r4000_tlb_load_handler(void)
1894{ 1896{
1895 u32 *p = handle_tlbl; 1897 u32 *p = handle_tlbl;
1898 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
1896 struct uasm_label *l = labels; 1899 struct uasm_label *l = labels;
1897 struct uasm_reloc *r = relocs; 1900 struct uasm_reloc *r = relocs;
1898 struct work_registers wr; 1901 struct work_registers wr;
1899 1902
1900 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1903 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
1901 memset(labels, 0, sizeof(labels)); 1904 memset(labels, 0, sizeof(labels));
1902 memset(relocs, 0, sizeof(relocs)); 1905 memset(relocs, 0, sizeof(relocs));
1903 1906
@@ -1935,6 +1938,19 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
1935 uasm_i_nop(&p); 1938 uasm_i_nop(&p);
1936 1939
1937 uasm_i_tlbr(&p); 1940 uasm_i_tlbr(&p);
1941
1942 switch (current_cpu_type()) {
1943 default:
1944 if (cpu_has_mips_r2) {
1945 uasm_i_ehb(&p);
1946
1947 case CPU_CAVIUM_OCTEON:
1948 case CPU_CAVIUM_OCTEON_PLUS:
1949 case CPU_CAVIUM_OCTEON2:
1950 break;
1951 }
1952 }
1953
1938 /* Examine entrylo 0 or 1 based on ptr. */ 1954 /* Examine entrylo 0 or 1 based on ptr. */
1939 if (use_bbit_insns()) { 1955 if (use_bbit_insns()) {
1940 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1956 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
@@ -1989,6 +2005,19 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
1989 uasm_i_nop(&p); 2005 uasm_i_nop(&p);
1990 2006
1991 uasm_i_tlbr(&p); 2007 uasm_i_tlbr(&p);
2008
2009 switch (current_cpu_type()) {
2010 default:
2011 if (cpu_has_mips_r2) {
2012 uasm_i_ehb(&p);
2013
2014 case CPU_CAVIUM_OCTEON:
2015 case CPU_CAVIUM_OCTEON_PLUS:
2016 case CPU_CAVIUM_OCTEON2:
2017 break;
2018 }
2019 }
2020
1992 /* Examine entrylo 0 or 1 based on ptr. */ 2021 /* Examine entrylo 0 or 1 based on ptr. */
1993 if (use_bbit_insns()) { 2022 if (use_bbit_insns()) {
1994 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 2023 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
@@ -2036,24 +2065,25 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
2036 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 2065 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
2037 uasm_i_nop(&p); 2066 uasm_i_nop(&p);
2038 2067
2039 if ((p - handle_tlbl) > FASTPATH_SIZE) 2068 if (p >= handle_tlbl_end)
2040 panic("TLB load handler fastpath space exceeded"); 2069 panic("TLB load handler fastpath space exceeded");
2041 2070
2042 uasm_resolve_relocs(relocs, labels); 2071 uasm_resolve_relocs(relocs, labels);
2043 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 2072 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
2044 (unsigned int)(p - handle_tlbl)); 2073 (unsigned int)(p - handle_tlbl));
2045 2074
2046 dump_handler("r4000_tlb_load", handle_tlbl, ARRAY_SIZE(handle_tlbl)); 2075 dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size);
2047} 2076}
2048 2077
2049static void __cpuinit build_r4000_tlb_store_handler(void) 2078static void build_r4000_tlb_store_handler(void)
2050{ 2079{
2051 u32 *p = handle_tlbs; 2080 u32 *p = handle_tlbs;
2081 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
2052 struct uasm_label *l = labels; 2082 struct uasm_label *l = labels;
2053 struct uasm_reloc *r = relocs; 2083 struct uasm_reloc *r = relocs;
2054 struct work_registers wr; 2084 struct work_registers wr;
2055 2085
2056 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 2086 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
2057 memset(labels, 0, sizeof(labels)); 2087 memset(labels, 0, sizeof(labels));
2058 memset(relocs, 0, sizeof(relocs)); 2088 memset(relocs, 0, sizeof(relocs));
2059 2089
@@ -2090,24 +2120,25 @@ static void __cpuinit build_r4000_tlb_store_handler(void)
2090 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2120 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2091 uasm_i_nop(&p); 2121 uasm_i_nop(&p);
2092 2122
2093 if ((p - handle_tlbs) > FASTPATH_SIZE) 2123 if (p >= handle_tlbs_end)
2094 panic("TLB store handler fastpath space exceeded"); 2124 panic("TLB store handler fastpath space exceeded");
2095 2125
2096 uasm_resolve_relocs(relocs, labels); 2126 uasm_resolve_relocs(relocs, labels);
2097 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2127 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
2098 (unsigned int)(p - handle_tlbs)); 2128 (unsigned int)(p - handle_tlbs));
2099 2129
2100 dump_handler("r4000_tlb_store", handle_tlbs, ARRAY_SIZE(handle_tlbs)); 2130 dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size);
2101} 2131}
2102 2132
2103static void __cpuinit build_r4000_tlb_modify_handler(void) 2133static void build_r4000_tlb_modify_handler(void)
2104{ 2134{
2105 u32 *p = handle_tlbm; 2135 u32 *p = handle_tlbm;
2136 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
2106 struct uasm_label *l = labels; 2137 struct uasm_label *l = labels;
2107 struct uasm_reloc *r = relocs; 2138 struct uasm_reloc *r = relocs;
2108 struct work_registers wr; 2139 struct work_registers wr;
2109 2140
2110 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 2141 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
2111 memset(labels, 0, sizeof(labels)); 2142 memset(labels, 0, sizeof(labels));
2112 memset(relocs, 0, sizeof(relocs)); 2143 memset(relocs, 0, sizeof(relocs));
2113 2144
@@ -2145,17 +2176,31 @@ static void __cpuinit build_r4000_tlb_modify_handler(void)
2145 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2176 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2146 uasm_i_nop(&p); 2177 uasm_i_nop(&p);
2147 2178
2148 if ((p - handle_tlbm) > FASTPATH_SIZE) 2179 if (p >= handle_tlbm_end)
2149 panic("TLB modify handler fastpath space exceeded"); 2180 panic("TLB modify handler fastpath space exceeded");
2150 2181
2151 uasm_resolve_relocs(relocs, labels); 2182 uasm_resolve_relocs(relocs, labels);
2152 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2183 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
2153 (unsigned int)(p - handle_tlbm)); 2184 (unsigned int)(p - handle_tlbm));
2154 2185
2155 dump_handler("r4000_tlb_modify", handle_tlbm, ARRAY_SIZE(handle_tlbm)); 2186 dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size);
2187}
2188
2189static void flush_tlb_handlers(void)
2190{
2191 local_flush_icache_range((unsigned long)handle_tlbl,
2192 (unsigned long)handle_tlbl_end);
2193 local_flush_icache_range((unsigned long)handle_tlbs,
2194 (unsigned long)handle_tlbs_end);
2195 local_flush_icache_range((unsigned long)handle_tlbm,
2196 (unsigned long)handle_tlbm_end);
2197#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
2198 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
2199 (unsigned long)tlbmiss_handler_setup_pgd_end);
2200#endif
2156} 2201}
2157 2202
2158void __cpuinit build_tlb_refill_handler(void) 2203void build_tlb_refill_handler(void)
2159{ 2204{
2160 /* 2205 /*
2161 * The refill handler is generated per-CPU, multi-node systems 2206 * The refill handler is generated per-CPU, multi-node systems
@@ -2187,6 +2232,7 @@ void __cpuinit build_tlb_refill_handler(void)
2187 build_r3000_tlb_load_handler(); 2232 build_r3000_tlb_load_handler();
2188 build_r3000_tlb_store_handler(); 2233 build_r3000_tlb_store_handler();
2189 build_r3000_tlb_modify_handler(); 2234 build_r3000_tlb_modify_handler();
2235 flush_tlb_handlers();
2190 run_once++; 2236 run_once++;
2191 } 2237 }
2192#else 2238#else
@@ -2214,23 +2260,10 @@ void __cpuinit build_tlb_refill_handler(void)
2214 build_r4000_tlb_modify_handler(); 2260 build_r4000_tlb_modify_handler();
2215 if (!cpu_has_local_ebase) 2261 if (!cpu_has_local_ebase)
2216 build_r4000_tlb_refill_handler(); 2262 build_r4000_tlb_refill_handler();
2263 flush_tlb_handlers();
2217 run_once++; 2264 run_once++;
2218 } 2265 }
2219 if (cpu_has_local_ebase) 2266 if (cpu_has_local_ebase)
2220 build_r4000_tlb_refill_handler(); 2267 build_r4000_tlb_refill_handler();
2221 } 2268 }
2222} 2269}
2223
2224void __cpuinit flush_tlb_handlers(void)
2225{
2226 local_flush_icache_range((unsigned long)handle_tlbl,
2227 (unsigned long)handle_tlbl + sizeof(handle_tlbl));
2228 local_flush_icache_range((unsigned long)handle_tlbs,
2229 (unsigned long)handle_tlbs + sizeof(handle_tlbs));
2230 local_flush_icache_range((unsigned long)handle_tlbm,
2231 (unsigned long)handle_tlbm + sizeof(handle_tlbm));
2232#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
2233 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd_array,
2234 (unsigned long)tlbmiss_handler_setup_pgd_array + sizeof(handle_tlbm));
2235#endif
2236}
diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c
index 162ee6d62788..060000fa653c 100644
--- a/arch/mips/mm/uasm-micromips.c
+++ b/arch/mips/mm/uasm-micromips.c
@@ -49,7 +49,7 @@
49 49
50#include "uasm.c" 50#include "uasm.c"
51 51
52static struct insn insn_table_MM[] __uasminitdata = { 52static struct insn insn_table_MM[] = {
53 { insn_addu, M(mm_pool32a_op, 0, 0, 0, 0, mm_addu32_op), RT | RS | RD }, 53 { insn_addu, M(mm_pool32a_op, 0, 0, 0, 0, mm_addu32_op), RT | RS | RD },
54 { insn_addiu, M(mm_addiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM }, 54 { insn_addiu, M(mm_addiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM },
55 { insn_and, M(mm_pool32a_op, 0, 0, 0, 0, mm_and_op), RT | RS | RD }, 55 { insn_and, M(mm_pool32a_op, 0, 0, 0, 0, mm_and_op), RT | RS | RD },
@@ -118,7 +118,7 @@ static struct insn insn_table_MM[] __uasminitdata = {
118 118
119#undef M 119#undef M
120 120
121static inline __uasminit u32 build_bimm(s32 arg) 121static inline u32 build_bimm(s32 arg)
122{ 122{
123 WARN(arg > 0xffff || arg < -0x10000, 123 WARN(arg > 0xffff || arg < -0x10000,
124 KERN_WARNING "Micro-assembler field overflow\n"); 124 KERN_WARNING "Micro-assembler field overflow\n");
@@ -128,7 +128,7 @@ static inline __uasminit u32 build_bimm(s32 arg)
128 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 1) & 0x7fff); 128 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 1) & 0x7fff);
129} 129}
130 130
131static inline __uasminit u32 build_jimm(u32 arg) 131static inline u32 build_jimm(u32 arg)
132{ 132{
133 133
134 WARN(arg & ~((JIMM_MASK << 2) | 1), 134 WARN(arg & ~((JIMM_MASK << 2) | 1),
@@ -141,7 +141,7 @@ static inline __uasminit u32 build_jimm(u32 arg)
141 * The order of opcode arguments is implicitly left to right, 141 * The order of opcode arguments is implicitly left to right,
142 * starting with RS and ending with FUNC or IMM. 142 * starting with RS and ending with FUNC or IMM.
143 */ 143 */
144static void __uasminit build_insn(u32 **buf, enum opcode opc, ...) 144static void build_insn(u32 **buf, enum opcode opc, ...)
145{ 145{
146 struct insn *ip = NULL; 146 struct insn *ip = NULL;
147 unsigned int i; 147 unsigned int i;
@@ -199,7 +199,7 @@ static void __uasminit build_insn(u32 **buf, enum opcode opc, ...)
199 (*buf)++; 199 (*buf)++;
200} 200}
201 201
202static inline void __uasminit 202static inline void
203__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab) 203__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
204{ 204{
205 long laddr = (long)lab->addr; 205 long laddr = (long)lab->addr;
diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c
index 5fcdd8fe3e83..0c724589854e 100644
--- a/arch/mips/mm/uasm-mips.c
+++ b/arch/mips/mm/uasm-mips.c
@@ -49,7 +49,7 @@
49 49
50#include "uasm.c" 50#include "uasm.c"
51 51
52static struct insn insn_table[] __uasminitdata = { 52static struct insn insn_table[] = {
53 { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, 53 { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
54 { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD }, 54 { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD },
55 { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM }, 55 { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
@@ -119,7 +119,7 @@ static struct insn insn_table[] __uasminitdata = {
119 119
120#undef M 120#undef M
121 121
122static inline __uasminit u32 build_bimm(s32 arg) 122static inline u32 build_bimm(s32 arg)
123{ 123{
124 WARN(arg > 0x1ffff || arg < -0x20000, 124 WARN(arg > 0x1ffff || arg < -0x20000,
125 KERN_WARNING "Micro-assembler field overflow\n"); 125 KERN_WARNING "Micro-assembler field overflow\n");
@@ -129,7 +129,7 @@ static inline __uasminit u32 build_bimm(s32 arg)
129 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff); 129 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
130} 130}
131 131
132static inline __uasminit u32 build_jimm(u32 arg) 132static inline u32 build_jimm(u32 arg)
133{ 133{
134 WARN(arg & ~(JIMM_MASK << 2), 134 WARN(arg & ~(JIMM_MASK << 2),
135 KERN_WARNING "Micro-assembler field overflow\n"); 135 KERN_WARNING "Micro-assembler field overflow\n");
@@ -141,7 +141,7 @@ static inline __uasminit u32 build_jimm(u32 arg)
141 * The order of opcode arguments is implicitly left to right, 141 * The order of opcode arguments is implicitly left to right,
142 * starting with RS and ending with FUNC or IMM. 142 * starting with RS and ending with FUNC or IMM.
143 */ 143 */
144static void __uasminit build_insn(u32 **buf, enum opcode opc, ...) 144static void build_insn(u32 **buf, enum opcode opc, ...)
145{ 145{
146 struct insn *ip = NULL; 146 struct insn *ip = NULL;
147 unsigned int i; 147 unsigned int i;
@@ -187,7 +187,7 @@ static void __uasminit build_insn(u32 **buf, enum opcode opc, ...)
187 (*buf)++; 187 (*buf)++;
188} 188}
189 189
190static inline void __uasminit 190static inline void
191__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab) 191__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
192{ 192{
193 long laddr = (long)lab->addr; 193 long laddr = (long)lab->addr;
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
index 7eb5e4355d25..b9d14b6c7f58 100644
--- a/arch/mips/mm/uasm.c
+++ b/arch/mips/mm/uasm.c
@@ -63,35 +63,35 @@ struct insn {
63 enum fields fields; 63 enum fields fields;
64}; 64};
65 65
66static inline __uasminit u32 build_rs(u32 arg) 66static inline u32 build_rs(u32 arg)
67{ 67{
68 WARN(arg & ~RS_MASK, KERN_WARNING "Micro-assembler field overflow\n"); 68 WARN(arg & ~RS_MASK, KERN_WARNING "Micro-assembler field overflow\n");
69 69
70 return (arg & RS_MASK) << RS_SH; 70 return (arg & RS_MASK) << RS_SH;
71} 71}
72 72
73static inline __uasminit u32 build_rt(u32 arg) 73static inline u32 build_rt(u32 arg)
74{ 74{
75 WARN(arg & ~RT_MASK, KERN_WARNING "Micro-assembler field overflow\n"); 75 WARN(arg & ~RT_MASK, KERN_WARNING "Micro-assembler field overflow\n");
76 76
77 return (arg & RT_MASK) << RT_SH; 77 return (arg & RT_MASK) << RT_SH;
78} 78}
79 79
80static inline __uasminit u32 build_rd(u32 arg) 80static inline u32 build_rd(u32 arg)
81{ 81{
82 WARN(arg & ~RD_MASK, KERN_WARNING "Micro-assembler field overflow\n"); 82 WARN(arg & ~RD_MASK, KERN_WARNING "Micro-assembler field overflow\n");
83 83
84 return (arg & RD_MASK) << RD_SH; 84 return (arg & RD_MASK) << RD_SH;
85} 85}
86 86
87static inline __uasminit u32 build_re(u32 arg) 87static inline u32 build_re(u32 arg)
88{ 88{
89 WARN(arg & ~RE_MASK, KERN_WARNING "Micro-assembler field overflow\n"); 89 WARN(arg & ~RE_MASK, KERN_WARNING "Micro-assembler field overflow\n");
90 90
91 return (arg & RE_MASK) << RE_SH; 91 return (arg & RE_MASK) << RE_SH;
92} 92}
93 93
94static inline __uasminit u32 build_simm(s32 arg) 94static inline u32 build_simm(s32 arg)
95{ 95{
96 WARN(arg > 0x7fff || arg < -0x8000, 96 WARN(arg > 0x7fff || arg < -0x8000,
97 KERN_WARNING "Micro-assembler field overflow\n"); 97 KERN_WARNING "Micro-assembler field overflow\n");
@@ -99,14 +99,14 @@ static inline __uasminit u32 build_simm(s32 arg)
99 return arg & 0xffff; 99 return arg & 0xffff;
100} 100}
101 101
102static inline __uasminit u32 build_uimm(u32 arg) 102static inline u32 build_uimm(u32 arg)
103{ 103{
104 WARN(arg & ~IMM_MASK, KERN_WARNING "Micro-assembler field overflow\n"); 104 WARN(arg & ~IMM_MASK, KERN_WARNING "Micro-assembler field overflow\n");
105 105
106 return arg & IMM_MASK; 106 return arg & IMM_MASK;
107} 107}
108 108
109static inline __uasminit u32 build_scimm(u32 arg) 109static inline u32 build_scimm(u32 arg)
110{ 110{
111 WARN(arg & ~SCIMM_MASK, 111 WARN(arg & ~SCIMM_MASK,
112 KERN_WARNING "Micro-assembler field overflow\n"); 112 KERN_WARNING "Micro-assembler field overflow\n");
@@ -114,21 +114,21 @@ static inline __uasminit u32 build_scimm(u32 arg)
114 return (arg & SCIMM_MASK) << SCIMM_SH; 114 return (arg & SCIMM_MASK) << SCIMM_SH;
115} 115}
116 116
117static inline __uasminit u32 build_func(u32 arg) 117static inline u32 build_func(u32 arg)
118{ 118{
119 WARN(arg & ~FUNC_MASK, KERN_WARNING "Micro-assembler field overflow\n"); 119 WARN(arg & ~FUNC_MASK, KERN_WARNING "Micro-assembler field overflow\n");
120 120
121 return arg & FUNC_MASK; 121 return arg & FUNC_MASK;
122} 122}
123 123
124static inline __uasminit u32 build_set(u32 arg) 124static inline u32 build_set(u32 arg)
125{ 125{
126 WARN(arg & ~SET_MASK, KERN_WARNING "Micro-assembler field overflow\n"); 126 WARN(arg & ~SET_MASK, KERN_WARNING "Micro-assembler field overflow\n");
127 127
128 return arg & SET_MASK; 128 return arg & SET_MASK;
129} 129}
130 130
131static void __uasminit build_insn(u32 **buf, enum opcode opc, ...); 131static void build_insn(u32 **buf, enum opcode opc, ...);
132 132
133#define I_u1u2u3(op) \ 133#define I_u1u2u3(op) \
134Ip_u1u2u3(op) \ 134Ip_u1u2u3(op) \
@@ -286,7 +286,7 @@ I_u3u1u2(_ldx)
286 286
287#ifdef CONFIG_CPU_CAVIUM_OCTEON 287#ifdef CONFIG_CPU_CAVIUM_OCTEON
288#include <asm/octeon/octeon.h> 288#include <asm/octeon/octeon.h>
289void __uasminit ISAFUNC(uasm_i_pref)(u32 **buf, unsigned int a, signed int b, 289void ISAFUNC(uasm_i_pref)(u32 **buf, unsigned int a, signed int b,
290 unsigned int c) 290 unsigned int c)
291{ 291{
292 if (OCTEON_IS_MODEL(OCTEON_CN63XX_PASS1_X) && a <= 24 && a != 5) 292 if (OCTEON_IS_MODEL(OCTEON_CN63XX_PASS1_X) && a <= 24 && a != 5)
@@ -304,7 +304,7 @@ I_u2s3u1(_pref)
304#endif 304#endif
305 305
306/* Handle labels. */ 306/* Handle labels. */
307void __uasminit ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, int lid) 307void ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, int lid)
308{ 308{
309 (*lab)->addr = addr; 309 (*lab)->addr = addr;
310 (*lab)->lab = lid; 310 (*lab)->lab = lid;
@@ -312,7 +312,7 @@ void __uasminit ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, in
312} 312}
313UASM_EXPORT_SYMBOL(ISAFUNC(uasm_build_label)); 313UASM_EXPORT_SYMBOL(ISAFUNC(uasm_build_label));
314 314
315int __uasminit ISAFUNC(uasm_in_compat_space_p)(long addr) 315int ISAFUNC(uasm_in_compat_space_p)(long addr)
316{ 316{
317 /* Is this address in 32bit compat space? */ 317 /* Is this address in 32bit compat space? */
318#ifdef CONFIG_64BIT 318#ifdef CONFIG_64BIT
@@ -323,7 +323,7 @@ int __uasminit ISAFUNC(uasm_in_compat_space_p)(long addr)
323} 323}
324UASM_EXPORT_SYMBOL(ISAFUNC(uasm_in_compat_space_p)); 324UASM_EXPORT_SYMBOL(ISAFUNC(uasm_in_compat_space_p));
325 325
326static int __uasminit uasm_rel_highest(long val) 326static int uasm_rel_highest(long val)
327{ 327{
328#ifdef CONFIG_64BIT 328#ifdef CONFIG_64BIT
329 return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000; 329 return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
@@ -332,7 +332,7 @@ static int __uasminit uasm_rel_highest(long val)
332#endif 332#endif
333} 333}
334 334
335static int __uasminit uasm_rel_higher(long val) 335static int uasm_rel_higher(long val)
336{ 336{
337#ifdef CONFIG_64BIT 337#ifdef CONFIG_64BIT
338 return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000; 338 return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
@@ -341,19 +341,19 @@ static int __uasminit uasm_rel_higher(long val)
341#endif 341#endif
342} 342}
343 343
344int __uasminit ISAFUNC(uasm_rel_hi)(long val) 344int ISAFUNC(uasm_rel_hi)(long val)
345{ 345{
346 return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000; 346 return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
347} 347}
348UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_hi)); 348UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_hi));
349 349
350int __uasminit ISAFUNC(uasm_rel_lo)(long val) 350int ISAFUNC(uasm_rel_lo)(long val)
351{ 351{
352 return ((val & 0xffff) ^ 0x8000) - 0x8000; 352 return ((val & 0xffff) ^ 0x8000) - 0x8000;
353} 353}
354UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_lo)); 354UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_lo));
355 355
356void __uasminit ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr) 356void ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr)
357{ 357{
358 if (!ISAFUNC(uasm_in_compat_space_p)(addr)) { 358 if (!ISAFUNC(uasm_in_compat_space_p)(addr)) {
359 ISAFUNC(uasm_i_lui)(buf, rs, uasm_rel_highest(addr)); 359 ISAFUNC(uasm_i_lui)(buf, rs, uasm_rel_highest(addr));
@@ -371,7 +371,7 @@ void __uasminit ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr)
371} 371}
372UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA_mostly)); 372UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA_mostly));
373 373
374void __uasminit ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr) 374void ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr)
375{ 375{
376 ISAFUNC(UASM_i_LA_mostly)(buf, rs, addr); 376 ISAFUNC(UASM_i_LA_mostly)(buf, rs, addr);
377 if (ISAFUNC(uasm_rel_lo(addr))) { 377 if (ISAFUNC(uasm_rel_lo(addr))) {
@@ -386,8 +386,7 @@ void __uasminit ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr)
386UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA)); 386UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA));
387 387
388/* Handle relocations. */ 388/* Handle relocations. */
389void __uasminit 389void ISAFUNC(uasm_r_mips_pc16)(struct uasm_reloc **rel, u32 *addr, int lid)
390ISAFUNC(uasm_r_mips_pc16)(struct uasm_reloc **rel, u32 *addr, int lid)
391{ 390{
392 (*rel)->addr = addr; 391 (*rel)->addr = addr;
393 (*rel)->type = R_MIPS_PC16; 392 (*rel)->type = R_MIPS_PC16;
@@ -396,11 +395,11 @@ ISAFUNC(uasm_r_mips_pc16)(struct uasm_reloc **rel, u32 *addr, int lid)
396} 395}
397UASM_EXPORT_SYMBOL(ISAFUNC(uasm_r_mips_pc16)); 396UASM_EXPORT_SYMBOL(ISAFUNC(uasm_r_mips_pc16));
398 397
399static inline void __uasminit 398static inline void __resolve_relocs(struct uasm_reloc *rel,
400__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab); 399 struct uasm_label *lab);
401 400
402void __uasminit 401void ISAFUNC(uasm_resolve_relocs)(struct uasm_reloc *rel,
403ISAFUNC(uasm_resolve_relocs)(struct uasm_reloc *rel, struct uasm_label *lab) 402 struct uasm_label *lab)
404{ 403{
405 struct uasm_label *l; 404 struct uasm_label *l;
406 405
@@ -411,8 +410,8 @@ ISAFUNC(uasm_resolve_relocs)(struct uasm_reloc *rel, struct uasm_label *lab)
411} 410}
412UASM_EXPORT_SYMBOL(ISAFUNC(uasm_resolve_relocs)); 411UASM_EXPORT_SYMBOL(ISAFUNC(uasm_resolve_relocs));
413 412
414void __uasminit 413void ISAFUNC(uasm_move_relocs)(struct uasm_reloc *rel, u32 *first, u32 *end,
415ISAFUNC(uasm_move_relocs)(struct uasm_reloc *rel, u32 *first, u32 *end, long off) 414 long off)
416{ 415{
417 for (; rel->lab != UASM_LABEL_INVALID; rel++) 416 for (; rel->lab != UASM_LABEL_INVALID; rel++)
418 if (rel->addr >= first && rel->addr < end) 417 if (rel->addr >= first && rel->addr < end)
@@ -420,8 +419,8 @@ ISAFUNC(uasm_move_relocs)(struct uasm_reloc *rel, u32 *first, u32 *end, long off
420} 419}
421UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_relocs)); 420UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_relocs));
422 421
423void __uasminit 422void ISAFUNC(uasm_move_labels)(struct uasm_label *lab, u32 *first, u32 *end,
424ISAFUNC(uasm_move_labels)(struct uasm_label *lab, u32 *first, u32 *end, long off) 423 long off)
425{ 424{
426 for (; lab->lab != UASM_LABEL_INVALID; lab++) 425 for (; lab->lab != UASM_LABEL_INVALID; lab++)
427 if (lab->addr >= first && lab->addr < end) 426 if (lab->addr >= first && lab->addr < end)
@@ -429,9 +428,8 @@ ISAFUNC(uasm_move_labels)(struct uasm_label *lab, u32 *first, u32 *end, long off
429} 428}
430UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_labels)); 429UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_labels));
431 430
432void __uasminit 431void ISAFUNC(uasm_copy_handler)(struct uasm_reloc *rel, struct uasm_label *lab,
433ISAFUNC(uasm_copy_handler)(struct uasm_reloc *rel, struct uasm_label *lab, u32 *first, 432 u32 *first, u32 *end, u32 *target)
434 u32 *end, u32 *target)
435{ 433{
436 long off = (long)(target - first); 434 long off = (long)(target - first);
437 435
@@ -442,7 +440,7 @@ ISAFUNC(uasm_copy_handler)(struct uasm_reloc *rel, struct uasm_label *lab, u32 *
442} 440}
443UASM_EXPORT_SYMBOL(ISAFUNC(uasm_copy_handler)); 441UASM_EXPORT_SYMBOL(ISAFUNC(uasm_copy_handler));
444 442
445int __uasminit ISAFUNC(uasm_insn_has_bdelay)(struct uasm_reloc *rel, u32 *addr) 443int ISAFUNC(uasm_insn_has_bdelay)(struct uasm_reloc *rel, u32 *addr)
446{ 444{
447 for (; rel->lab != UASM_LABEL_INVALID; rel++) { 445 for (; rel->lab != UASM_LABEL_INVALID; rel++) {
448 if (rel->addr == addr 446 if (rel->addr == addr
@@ -456,83 +454,79 @@ int __uasminit ISAFUNC(uasm_insn_has_bdelay)(struct uasm_reloc *rel, u32 *addr)
456UASM_EXPORT_SYMBOL(ISAFUNC(uasm_insn_has_bdelay)); 454UASM_EXPORT_SYMBOL(ISAFUNC(uasm_insn_has_bdelay));
457 455
458/* Convenience functions for labeled branches. */ 456/* Convenience functions for labeled branches. */
459void __uasminit 457void ISAFUNC(uasm_il_bltz)(u32 **p, struct uasm_reloc **r, unsigned int reg,
460ISAFUNC(uasm_il_bltz)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) 458 int lid)
461{ 459{
462 uasm_r_mips_pc16(r, *p, lid); 460 uasm_r_mips_pc16(r, *p, lid);
463 ISAFUNC(uasm_i_bltz)(p, reg, 0); 461 ISAFUNC(uasm_i_bltz)(p, reg, 0);
464} 462}
465UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bltz)); 463UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bltz));
466 464
467void __uasminit 465void ISAFUNC(uasm_il_b)(u32 **p, struct uasm_reloc **r, int lid)
468ISAFUNC(uasm_il_b)(u32 **p, struct uasm_reloc **r, int lid)
469{ 466{
470 uasm_r_mips_pc16(r, *p, lid); 467 uasm_r_mips_pc16(r, *p, lid);
471 ISAFUNC(uasm_i_b)(p, 0); 468 ISAFUNC(uasm_i_b)(p, 0);
472} 469}
473UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_b)); 470UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_b));
474 471
475void __uasminit 472void ISAFUNC(uasm_il_beqz)(u32 **p, struct uasm_reloc **r, unsigned int reg,
476ISAFUNC(uasm_il_beqz)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) 473 int lid)
477{ 474{
478 uasm_r_mips_pc16(r, *p, lid); 475 uasm_r_mips_pc16(r, *p, lid);
479 ISAFUNC(uasm_i_beqz)(p, reg, 0); 476 ISAFUNC(uasm_i_beqz)(p, reg, 0);
480} 477}
481UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqz)); 478UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqz));
482 479
483void __uasminit 480void ISAFUNC(uasm_il_beqzl)(u32 **p, struct uasm_reloc **r, unsigned int reg,
484ISAFUNC(uasm_il_beqzl)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) 481 int lid)
485{ 482{
486 uasm_r_mips_pc16(r, *p, lid); 483 uasm_r_mips_pc16(r, *p, lid);
487 ISAFUNC(uasm_i_beqzl)(p, reg, 0); 484 ISAFUNC(uasm_i_beqzl)(p, reg, 0);
488} 485}
489UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqzl)); 486UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqzl));
490 487
491void __uasminit 488void ISAFUNC(uasm_il_bne)(u32 **p, struct uasm_reloc **r, unsigned int reg1,
492ISAFUNC(uasm_il_bne)(u32 **p, struct uasm_reloc **r, unsigned int reg1, 489 unsigned int reg2, int lid)
493 unsigned int reg2, int lid)
494{ 490{
495 uasm_r_mips_pc16(r, *p, lid); 491 uasm_r_mips_pc16(r, *p, lid);
496 ISAFUNC(uasm_i_bne)(p, reg1, reg2, 0); 492 ISAFUNC(uasm_i_bne)(p, reg1, reg2, 0);
497} 493}
498UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bne)); 494UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bne));
499 495
500void __uasminit 496void ISAFUNC(uasm_il_bnez)(u32 **p, struct uasm_reloc **r, unsigned int reg,
501ISAFUNC(uasm_il_bnez)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) 497 int lid)
502{ 498{
503 uasm_r_mips_pc16(r, *p, lid); 499 uasm_r_mips_pc16(r, *p, lid);
504 ISAFUNC(uasm_i_bnez)(p, reg, 0); 500 ISAFUNC(uasm_i_bnez)(p, reg, 0);
505} 501}
506UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bnez)); 502UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bnez));
507 503
508void __uasminit 504void ISAFUNC(uasm_il_bgezl)(u32 **p, struct uasm_reloc **r, unsigned int reg,
509ISAFUNC(uasm_il_bgezl)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) 505 int lid)
510{ 506{
511 uasm_r_mips_pc16(r, *p, lid); 507 uasm_r_mips_pc16(r, *p, lid);
512 ISAFUNC(uasm_i_bgezl)(p, reg, 0); 508 ISAFUNC(uasm_i_bgezl)(p, reg, 0);
513} 509}
514UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgezl)); 510UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgezl));
515 511
516void __uasminit 512void ISAFUNC(uasm_il_bgez)(u32 **p, struct uasm_reloc **r, unsigned int reg,
517ISAFUNC(uasm_il_bgez)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) 513 int lid)
518{ 514{
519 uasm_r_mips_pc16(r, *p, lid); 515 uasm_r_mips_pc16(r, *p, lid);
520 ISAFUNC(uasm_i_bgez)(p, reg, 0); 516 ISAFUNC(uasm_i_bgez)(p, reg, 0);
521} 517}
522UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgez)); 518UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgez));
523 519
524void __uasminit 520void ISAFUNC(uasm_il_bbit0)(u32 **p, struct uasm_reloc **r, unsigned int reg,
525ISAFUNC(uasm_il_bbit0)(u32 **p, struct uasm_reloc **r, unsigned int reg, 521 unsigned int bit, int lid)
526 unsigned int bit, int lid)
527{ 522{
528 uasm_r_mips_pc16(r, *p, lid); 523 uasm_r_mips_pc16(r, *p, lid);
529 ISAFUNC(uasm_i_bbit0)(p, reg, bit, 0); 524 ISAFUNC(uasm_i_bbit0)(p, reg, bit, 0);
530} 525}
531UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bbit0)); 526UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bbit0));
532 527
533void __uasminit 528void ISAFUNC(uasm_il_bbit1)(u32 **p, struct uasm_reloc **r, unsigned int reg,
534ISAFUNC(uasm_il_bbit1)(u32 **p, struct uasm_reloc **r, unsigned int reg, 529 unsigned int bit, int lid)
535 unsigned int bit, int lid)
536{ 530{
537 uasm_r_mips_pc16(r, *p, lid); 531 uasm_r_mips_pc16(r, *p, lid);
538 ISAFUNC(uasm_i_bbit1)(p, reg, bit, 0); 532 ISAFUNC(uasm_i_bbit1)(p, reg, bit, 0);
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-12 17:45:52 -0400