diff options
author | David Daney <ddaney@caviumnetworks.com> | 2009-05-27 20:47:44 -0400 |
---|---|---|
committer | Ralf Baechle <ralf@linux-mips.org> | 2009-06-17 06:06:30 -0400 |
commit | fd062c847a8cea2821347d7e18165dfa658f7dce (patch) | |
tree | 95410c5460f7d153a9c9b15184ea52e2059427c3 /arch/mips | |
parent | dd7943920b492d9d8a79080fe05e25ecd7e10bc3 (diff) |
MIPS: TLB support for hugetlbfs.
The TLB handlers need to check for huge pages and give them special
handling. Huge pages consist of two contiguous sub-pages of physical
memory.
* Loading entrylo0 and entrylo1 need to be handled specially.
* The page mask must be set for huge pages and then restored after
writing the TLB entries.
* The PTE for huge pages resides in the PMD, we halt traversal of the
tables there.
Signed-off-by: David Daney <ddaney@caviumnetworks.com>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
Diffstat (limited to 'arch/mips')
-rw-r--r-- | arch/mips/mm/tlb-r4k.c | 43 | ||||
-rw-r--r-- | arch/mips/mm/tlbex.c | 165 |
2 files changed, 196 insertions, 12 deletions
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 892be426787c..f60fe513eb60 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c | |||
@@ -11,6 +11,7 @@ | |||
11 | #include <linux/init.h> | 11 | #include <linux/init.h> |
12 | #include <linux/sched.h> | 12 | #include <linux/sched.h> |
13 | #include <linux/mm.h> | 13 | #include <linux/mm.h> |
14 | #include <linux/hugetlb.h> | ||
14 | 15 | ||
15 | #include <asm/cpu.h> | 16 | #include <asm/cpu.h> |
16 | #include <asm/bootinfo.h> | 17 | #include <asm/bootinfo.h> |
@@ -295,21 +296,41 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) | |||
295 | pudp = pud_offset(pgdp, address); | 296 | pudp = pud_offset(pgdp, address); |
296 | pmdp = pmd_offset(pudp, address); | 297 | pmdp = pmd_offset(pudp, address); |
297 | idx = read_c0_index(); | 298 | idx = read_c0_index(); |
298 | ptep = pte_offset_map(pmdp, address); | 299 | #ifdef CONFIG_HUGETLB_PAGE |
300 | /* this could be a huge page */ | ||
301 | if (pmd_huge(*pmdp)) { | ||
302 | unsigned long lo; | ||
303 | write_c0_pagemask(PM_HUGE_MASK); | ||
304 | ptep = (pte_t *)pmdp; | ||
305 | lo = pte_val(*ptep) >> 6; | ||
306 | write_c0_entrylo0(lo); | ||
307 | write_c0_entrylo1(lo + (HPAGE_SIZE >> 7)); | ||
308 | |||
309 | mtc0_tlbw_hazard(); | ||
310 | if (idx < 0) | ||
311 | tlb_write_random(); | ||
312 | else | ||
313 | tlb_write_indexed(); | ||
314 | write_c0_pagemask(PM_DEFAULT_MASK); | ||
315 | } else | ||
316 | #endif | ||
317 | { | ||
318 | ptep = pte_offset_map(pmdp, address); | ||
299 | 319 | ||
300 | #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) | 320 | #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) |
301 | write_c0_entrylo0(ptep->pte_high); | 321 | write_c0_entrylo0(ptep->pte_high); |
302 | ptep++; | 322 | ptep++; |
303 | write_c0_entrylo1(ptep->pte_high); | 323 | write_c0_entrylo1(ptep->pte_high); |
304 | #else | 324 | #else |
305 | write_c0_entrylo0(pte_val(*ptep++) >> 6); | 325 | write_c0_entrylo0(pte_val(*ptep++) >> 6); |
306 | write_c0_entrylo1(pte_val(*ptep) >> 6); | 326 | write_c0_entrylo1(pte_val(*ptep) >> 6); |
307 | #endif | 327 | #endif |
308 | mtc0_tlbw_hazard(); | 328 | mtc0_tlbw_hazard(); |
309 | if (idx < 0) | 329 | if (idx < 0) |
310 | tlb_write_random(); | 330 | tlb_write_random(); |
311 | else | 331 | else |
312 | tlb_write_indexed(); | 332 | tlb_write_indexed(); |
333 | } | ||
313 | tlbw_use_hazard(); | 334 | tlbw_use_hazard(); |
314 | FLUSH_ITLB_VM(vma); | 335 | FLUSH_ITLB_VM(vma); |
315 | EXIT_CRITICAL(flags); | 336 | EXIT_CRITICAL(flags); |
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 62fbd0d89aeb..8f606ead826e 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c | |||
@@ -8,6 +8,7 @@ | |||
8 | * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer | 8 | * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer |
9 | * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki | 9 | * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki |
10 | * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) | 10 | * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) |
11 | * Copyright (C) 2008, 2009 Cavium Networks, Inc. | ||
11 | * | 12 | * |
12 | * ... and the days got worse and worse and now you see | 13 | * ... and the days got worse and worse and now you see |
13 | * I've gone completly out of my mind. | 14 | * I've gone completly out of my mind. |
@@ -83,6 +84,9 @@ enum label_id { | |||
83 | label_nopage_tlbm, | 84 | label_nopage_tlbm, |
84 | label_smp_pgtable_change, | 85 | label_smp_pgtable_change, |
85 | label_r3000_write_probe_fail, | 86 | label_r3000_write_probe_fail, |
87 | #ifdef CONFIG_HUGETLB_PAGE | ||
88 | label_tlb_huge_update, | ||
89 | #endif | ||
86 | }; | 90 | }; |
87 | 91 | ||
88 | UASM_L_LA(_second_part) | 92 | UASM_L_LA(_second_part) |
@@ -99,6 +103,9 @@ UASM_L_LA(_nopage_tlbs) | |||
99 | UASM_L_LA(_nopage_tlbm) | 103 | UASM_L_LA(_nopage_tlbm) |
100 | UASM_L_LA(_smp_pgtable_change) | 104 | UASM_L_LA(_smp_pgtable_change) |
101 | UASM_L_LA(_r3000_write_probe_fail) | 105 | UASM_L_LA(_r3000_write_probe_fail) |
106 | #ifdef CONFIG_HUGETLB_PAGE | ||
107 | UASM_L_LA(_tlb_huge_update) | ||
108 | #endif | ||
102 | 109 | ||
103 | /* | 110 | /* |
104 | * For debug purposes. | 111 | * For debug purposes. |
@@ -126,6 +133,7 @@ static inline void dump_handler(const u32 *handler, int count) | |||
126 | #define C0_TCBIND 2, 2 | 133 | #define C0_TCBIND 2, 2 |
127 | #define C0_ENTRYLO1 3, 0 | 134 | #define C0_ENTRYLO1 3, 0 |
128 | #define C0_CONTEXT 4, 0 | 135 | #define C0_CONTEXT 4, 0 |
136 | #define C0_PAGEMASK 5, 0 | ||
129 | #define C0_BADVADDR 8, 0 | 137 | #define C0_BADVADDR 8, 0 |
130 | #define C0_ENTRYHI 10, 0 | 138 | #define C0_ENTRYHI 10, 0 |
131 | #define C0_EPC 14, 0 | 139 | #define C0_EPC 14, 0 |
@@ -383,6 +391,98 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, | |||
383 | } | 391 | } |
384 | } | 392 | } |
385 | 393 | ||
394 | #ifdef CONFIG_HUGETLB_PAGE | ||
395 | static __cpuinit void build_huge_tlb_write_entry(u32 **p, | ||
396 | struct uasm_label **l, | ||
397 | struct uasm_reloc **r, | ||
398 | unsigned int tmp, | ||
399 | enum tlb_write_entry wmode) | ||
400 | { | ||
401 | /* Set huge page tlb entry size */ | ||
402 | uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); | ||
403 | uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); | ||
404 | uasm_i_mtc0(p, tmp, C0_PAGEMASK); | ||
405 | |||
406 | build_tlb_write_entry(p, l, r, wmode); | ||
407 | |||
408 | /* Reset default page size */ | ||
409 | if (PM_DEFAULT_MASK >> 16) { | ||
410 | uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); | ||
411 | uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); | ||
412 | uasm_il_b(p, r, label_leave); | ||
413 | uasm_i_mtc0(p, tmp, C0_PAGEMASK); | ||
414 | } else if (PM_DEFAULT_MASK) { | ||
415 | uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); | ||
416 | uasm_il_b(p, r, label_leave); | ||
417 | uasm_i_mtc0(p, tmp, C0_PAGEMASK); | ||
418 | } else { | ||
419 | uasm_il_b(p, r, label_leave); | ||
420 | uasm_i_mtc0(p, 0, C0_PAGEMASK); | ||
421 | } | ||
422 | } | ||
423 | |||
424 | /* | ||
425 | * Check if Huge PTE is present, if so then jump to LABEL. | ||
426 | */ | ||
427 | static void __cpuinit | ||
428 | build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, | ||
429 | unsigned int pmd, int lid) | ||
430 | { | ||
431 | UASM_i_LW(p, tmp, 0, pmd); | ||
432 | uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); | ||
433 | uasm_il_bnez(p, r, tmp, lid); | ||
434 | } | ||
435 | |||
436 | static __cpuinit void build_huge_update_entries(u32 **p, | ||
437 | unsigned int pte, | ||
438 | unsigned int tmp) | ||
439 | { | ||
440 | int small_sequence; | ||
441 | |||
442 | /* | ||
443 | * A huge PTE describes an area the size of the | ||
444 | * configured huge page size. This is twice the | ||
445 | * of the large TLB entry size we intend to use. | ||
446 | * A TLB entry half the size of the configured | ||
447 | * huge page size is configured into entrylo0 | ||
448 | * and entrylo1 to cover the contiguous huge PTE | ||
449 | * address space. | ||
450 | */ | ||
451 | small_sequence = (HPAGE_SIZE >> 7) < 0x10000; | ||
452 | |||
453 | /* We can clobber tmp. It isn't used after this.*/ | ||
454 | if (!small_sequence) | ||
455 | uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); | ||
456 | |||
457 | UASM_i_SRL(p, pte, pte, 6); /* convert to entrylo */ | ||
458 | uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* load it */ | ||
459 | /* convert to entrylo1 */ | ||
460 | if (small_sequence) | ||
461 | UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); | ||
462 | else | ||
463 | UASM_i_ADDU(p, pte, pte, tmp); | ||
464 | |||
465 | uasm_i_mtc0(p, pte, C0_ENTRYLO1); /* load it */ | ||
466 | } | ||
467 | |||
468 | static __cpuinit void build_huge_handler_tail(u32 **p, | ||
469 | struct uasm_reloc **r, | ||
470 | struct uasm_label **l, | ||
471 | unsigned int pte, | ||
472 | unsigned int ptr) | ||
473 | { | ||
474 | #ifdef CONFIG_SMP | ||
475 | UASM_i_SC(p, pte, 0, ptr); | ||
476 | uasm_il_beqz(p, r, pte, label_tlb_huge_update); | ||
477 | UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ | ||
478 | #else | ||
479 | UASM_i_SW(p, pte, 0, ptr); | ||
480 | #endif | ||
481 | build_huge_update_entries(p, pte, ptr); | ||
482 | build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed); | ||
483 | } | ||
484 | #endif /* CONFIG_HUGETLB_PAGE */ | ||
485 | |||
386 | #ifdef CONFIG_64BIT | 486 | #ifdef CONFIG_64BIT |
387 | /* | 487 | /* |
388 | * TMP and PTR are scratch. | 488 | * TMP and PTR are scratch. |
@@ -689,12 +789,23 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) | |||
689 | build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ | 789 | build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ |
690 | #endif | 790 | #endif |
691 | 791 | ||
792 | #ifdef CONFIG_HUGETLB_PAGE | ||
793 | build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); | ||
794 | #endif | ||
795 | |||
692 | build_get_ptep(&p, K0, K1); | 796 | build_get_ptep(&p, K0, K1); |
693 | build_update_entries(&p, K0, K1); | 797 | build_update_entries(&p, K0, K1); |
694 | build_tlb_write_entry(&p, &l, &r, tlb_random); | 798 | build_tlb_write_entry(&p, &l, &r, tlb_random); |
695 | uasm_l_leave(&l, p); | 799 | uasm_l_leave(&l, p); |
696 | uasm_i_eret(&p); /* return from trap */ | 800 | uasm_i_eret(&p); /* return from trap */ |
697 | 801 | ||
802 | #ifdef CONFIG_HUGETLB_PAGE | ||
803 | uasm_l_tlb_huge_update(&l, p); | ||
804 | UASM_i_LW(&p, K0, 0, K1); | ||
805 | build_huge_update_entries(&p, K0, K1); | ||
806 | build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random); | ||
807 | #endif | ||
808 | |||
698 | #ifdef CONFIG_64BIT | 809 | #ifdef CONFIG_64BIT |
699 | build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); | 810 | build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); |
700 | #endif | 811 | #endif |
@@ -733,7 +844,9 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) | |||
733 | uasm_copy_handler(relocs, labels, tlb_handler, p, f); | 844 | uasm_copy_handler(relocs, labels, tlb_handler, p, f); |
734 | final_len = p - tlb_handler; | 845 | final_len = p - tlb_handler; |
735 | } else { | 846 | } else { |
736 | #ifdef MODULE_START | 847 | #if defined(CONFIG_HUGETLB_PAGE) |
848 | const enum label_id ls = label_tlb_huge_update; | ||
849 | #elif defined(MODULE_START) | ||
737 | const enum label_id ls = label_module_alloc; | 850 | const enum label_id ls = label_module_alloc; |
738 | #else | 851 | #else |
739 | const enum label_id ls = label_vmalloc; | 852 | const enum label_id ls = label_vmalloc; |
@@ -1130,6 +1243,15 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, | |||
1130 | build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ | 1243 | build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ |
1131 | #endif | 1244 | #endif |
1132 | 1245 | ||
1246 | #ifdef CONFIG_HUGETLB_PAGE | ||
1247 | /* | ||
1248 | * For huge tlb entries, pmd doesn't contain an address but | ||
1249 | * instead contains the tlb pte. Check the PAGE_HUGE bit and | ||
1250 | * see if we need to jump to huge tlb processing. | ||
1251 | */ | ||
1252 | build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update); | ||
1253 | #endif | ||
1254 | |||
1133 | UASM_i_MFC0(p, pte, C0_BADVADDR); | 1255 | UASM_i_MFC0(p, pte, C0_BADVADDR); |
1134 | UASM_i_LW(p, ptr, 0, ptr); | 1256 | UASM_i_LW(p, ptr, 0, ptr); |
1135 | UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); | 1257 | UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); |
@@ -1187,6 +1309,19 @@ static void __cpuinit build_r4000_tlb_load_handler(void) | |||
1187 | build_make_valid(&p, &r, K0, K1); | 1309 | build_make_valid(&p, &r, K0, K1); |
1188 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); | 1310 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); |
1189 | 1311 | ||
1312 | #ifdef CONFIG_HUGETLB_PAGE | ||
1313 | /* | ||
1314 | * This is the entry point when build_r4000_tlbchange_handler_head | ||
1315 | * spots a huge page. | ||
1316 | */ | ||
1317 | uasm_l_tlb_huge_update(&l, p); | ||
1318 | iPTE_LW(&p, K0, K1); | ||
1319 | build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); | ||
1320 | build_tlb_probe_entry(&p); | ||
1321 | uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID)); | ||
1322 | build_huge_handler_tail(&p, &r, &l, K0, K1); | ||
1323 | #endif | ||
1324 | |||
1190 | uasm_l_nopage_tlbl(&l, p); | 1325 | uasm_l_nopage_tlbl(&l, p); |
1191 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); | 1326 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); |
1192 | uasm_i_nop(&p); | 1327 | uasm_i_nop(&p); |
@@ -1218,6 +1353,20 @@ static void __cpuinit build_r4000_tlb_store_handler(void) | |||
1218 | build_make_write(&p, &r, K0, K1); | 1353 | build_make_write(&p, &r, K0, K1); |
1219 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); | 1354 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); |
1220 | 1355 | ||
1356 | #ifdef CONFIG_HUGETLB_PAGE | ||
1357 | /* | ||
1358 | * This is the entry point when | ||
1359 | * build_r4000_tlbchange_handler_head spots a huge page. | ||
1360 | */ | ||
1361 | uasm_l_tlb_huge_update(&l, p); | ||
1362 | iPTE_LW(&p, K0, K1); | ||
1363 | build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); | ||
1364 | build_tlb_probe_entry(&p); | ||
1365 | uasm_i_ori(&p, K0, K0, | ||
1366 | _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); | ||
1367 | build_huge_handler_tail(&p, &r, &l, K0, K1); | ||
1368 | #endif | ||
1369 | |||
1221 | uasm_l_nopage_tlbs(&l, p); | 1370 | uasm_l_nopage_tlbs(&l, p); |
1222 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); | 1371 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); |
1223 | uasm_i_nop(&p); | 1372 | uasm_i_nop(&p); |
@@ -1250,6 +1399,20 @@ static void __cpuinit build_r4000_tlb_modify_handler(void) | |||
1250 | build_make_write(&p, &r, K0, K1); | 1399 | build_make_write(&p, &r, K0, K1); |
1251 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); | 1400 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); |
1252 | 1401 | ||
1402 | #ifdef CONFIG_HUGETLB_PAGE | ||
1403 | /* | ||
1404 | * This is the entry point when | ||
1405 | * build_r4000_tlbchange_handler_head spots a huge page. | ||
1406 | */ | ||
1407 | uasm_l_tlb_huge_update(&l, p); | ||
1408 | iPTE_LW(&p, K0, K1); | ||
1409 | build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); | ||
1410 | build_tlb_probe_entry(&p); | ||
1411 | uasm_i_ori(&p, K0, K0, | ||
1412 | _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); | ||
1413 | build_huge_handler_tail(&p, &r, &l, K0, K1); | ||
1414 | #endif | ||
1415 | |||
1253 | uasm_l_nopage_tlbm(&l, p); | 1416 | uasm_l_nopage_tlbm(&l, p); |
1254 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); | 1417 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); |
1255 | uasm_i_nop(&p); | 1418 | uasm_i_nop(&p); |