aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm
diff options
context:
space:
mode:
authorDavid Gibson <david@gibson.dropbear.id.au>2009-10-26 15:24:31 -0400
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>2009-10-30 02:20:59 -0400
commit883a3e523672ebba2ec3969837ba02af4f70fae2 (patch)
treeff8000aab5bc56348bd1b16ac904fe3a9b660b4a /arch/powerpc/mm
parentd1837cba5d5d5458c09f0a2849db2d3c203cb8e9 (diff)
powerpc/mm: Split hash MMU specific hugepage code into a new file
This patch separates the parts of hugetlbpage.c which are inherently specific to the hash MMU into a new hugelbpage-hash64.c file. Signed-off-by: David Gibson <dwg@au1.ibm.com> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Diffstat (limited to 'arch/powerpc/mm')
-rw-r--r--arch/powerpc/mm/Makefile5
-rw-r--r--arch/powerpc/mm/hugetlbpage-hash64.c167
-rw-r--r--arch/powerpc/mm/hugetlbpage.c168
3 files changed, 173 insertions, 167 deletions
diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile
index 6fb8fc8d2fea..ce68708bbad5 100644
--- a/arch/powerpc/mm/Makefile
+++ b/arch/powerpc/mm/Makefile
@@ -28,7 +28,10 @@ obj-$(CONFIG_44x) += 44x_mmu.o
28obj-$(CONFIG_FSL_BOOKE) += fsl_booke_mmu.o 28obj-$(CONFIG_FSL_BOOKE) += fsl_booke_mmu.o
29obj-$(CONFIG_NEED_MULTIPLE_NODES) += numa.o 29obj-$(CONFIG_NEED_MULTIPLE_NODES) += numa.o
30obj-$(CONFIG_PPC_MM_SLICES) += slice.o 30obj-$(CONFIG_PPC_MM_SLICES) += slice.o
31obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o 31ifeq ($(CONFIG_HUGETLB_PAGE),y)
32obj-y += hugetlbpage.o
33obj-$(CONFIG_PPC_STD_MMU_64) += hugetlbpage-hash64.o
34endif
32obj-$(CONFIG_PPC_SUBPAGE_PROT) += subpage-prot.o 35obj-$(CONFIG_PPC_SUBPAGE_PROT) += subpage-prot.o
33obj-$(CONFIG_NOT_COHERENT_CACHE) += dma-noncoherent.o 36obj-$(CONFIG_NOT_COHERENT_CACHE) += dma-noncoherent.o
34obj-$(CONFIG_HIGHMEM) += highmem.o 37obj-$(CONFIG_HIGHMEM) += highmem.o
diff --git a/arch/powerpc/mm/hugetlbpage-hash64.c b/arch/powerpc/mm/hugetlbpage-hash64.c
new file mode 100644
index 000000000000..1508ffc1e1e1
--- /dev/null
+++ b/arch/powerpc/mm/hugetlbpage-hash64.c
@@ -0,0 +1,167 @@
1/*
2 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 *
6 * Based on the IA-32 version:
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
8 */
9
10#include <linux/mm.h>
11#include <linux/hugetlb.h>
12#include <asm/pgtable.h>
13#include <asm/pgalloc.h>
14#include <asm/cacheflush.h>
15#include <asm/machdep.h>
16
17/*
18 * Called by asm hashtable.S for doing lazy icache flush
19 */
20static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags,
21 pte_t pte, int trap, unsigned long sz)
22{
23 struct page *page;
24 int i;
25
26 if (!pfn_valid(pte_pfn(pte)))
27 return rflags;
28
29 page = pte_page(pte);
30
31 /* page is dirty */
32 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
33 if (trap == 0x400) {
34 for (i = 0; i < (sz / PAGE_SIZE); i++)
35 __flush_dcache_icache(page_address(page+i));
36 set_bit(PG_arch_1, &page->flags);
37 } else {
38 rflags |= HPTE_R_N;
39 }
40 }
41 return rflags;
42}
43
44int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
45 pte_t *ptep, unsigned long trap, int local, int ssize,
46 unsigned int shift, unsigned int mmu_psize)
47{
48 unsigned long old_pte, new_pte;
49 unsigned long va, rflags, pa, sz;
50 long slot;
51 int err = 1;
52
53 BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
54
55 /* Search the Linux page table for a match with va */
56 va = hpt_va(ea, vsid, ssize);
57
58 /*
59 * Check the user's access rights to the page. If access should be
60 * prevented then send the problem up to do_page_fault.
61 */
62 if (unlikely(access & ~pte_val(*ptep)))
63 goto out;
64 /*
65 * At this point, we have a pte (old_pte) which can be used to build
66 * or update an HPTE. There are 2 cases:
67 *
68 * 1. There is a valid (present) pte with no associated HPTE (this is
69 * the most common case)
70 * 2. There is a valid (present) pte with an associated HPTE. The
71 * current values of the pp bits in the HPTE prevent access
72 * because we are doing software DIRTY bit management and the
73 * page is currently not DIRTY.
74 */
75
76
77 do {
78 old_pte = pte_val(*ptep);
79 if (old_pte & _PAGE_BUSY)
80 goto out;
81 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
82 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
83 old_pte, new_pte));
84
85 rflags = 0x2 | (!(new_pte & _PAGE_RW));
86 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
87 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
88 sz = ((1UL) << shift);
89 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
90 /* No CPU has hugepages but lacks no execute, so we
91 * don't need to worry about that case */
92 rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte),
93 trap, sz);
94
95 /* Check if pte already has an hpte (case 2) */
96 if (unlikely(old_pte & _PAGE_HASHPTE)) {
97 /* There MIGHT be an HPTE for this pte */
98 unsigned long hash, slot;
99
100 hash = hpt_hash(va, shift, ssize);
101 if (old_pte & _PAGE_F_SECOND)
102 hash = ~hash;
103 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
104 slot += (old_pte & _PAGE_F_GIX) >> 12;
105
106 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize,
107 ssize, local) == -1)
108 old_pte &= ~_PAGE_HPTEFLAGS;
109 }
110
111 if (likely(!(old_pte & _PAGE_HASHPTE))) {
112 unsigned long hash = hpt_hash(va, shift, ssize);
113 unsigned long hpte_group;
114
115 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
116
117repeat:
118 hpte_group = ((hash & htab_hash_mask) *
119 HPTES_PER_GROUP) & ~0x7UL;
120
121 /* clear HPTE slot informations in new PTE */
122#ifdef CONFIG_PPC_64K_PAGES
123 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
124#else
125 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
126#endif
127 /* Add in WIMG bits */
128 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
129 _PAGE_COHERENT | _PAGE_GUARDED));
130
131 /* Insert into the hash table, primary slot */
132 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
133 mmu_psize, ssize);
134
135 /* Primary is full, try the secondary */
136 if (unlikely(slot == -1)) {
137 hpte_group = ((~hash & htab_hash_mask) *
138 HPTES_PER_GROUP) & ~0x7UL;
139 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
140 HPTE_V_SECONDARY,
141 mmu_psize, ssize);
142 if (slot == -1) {
143 if (mftb() & 0x1)
144 hpte_group = ((hash & htab_hash_mask) *
145 HPTES_PER_GROUP)&~0x7UL;
146
147 ppc_md.hpte_remove(hpte_group);
148 goto repeat;
149 }
150 }
151
152 if (unlikely(slot == -2))
153 panic("hash_huge_page: pte_insert failed\n");
154
155 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
156 }
157
158 /*
159 * No need to use ldarx/stdcx here
160 */
161 *ptep = __pte(new_pte & ~_PAGE_BUSY);
162
163 err = 0;
164
165 out:
166 return err;
167}
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index a7161c07886d..1bf065546fa1 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -7,29 +7,17 @@
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com> 7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
8 */ 8 */
9 9
10#include <linux/init.h>
11#include <linux/fs.h>
12#include <linux/mm.h> 10#include <linux/mm.h>
11#include <linux/io.h>
13#include <linux/hugetlb.h> 12#include <linux/hugetlb.h>
14#include <linux/pagemap.h> 13#include <asm/pgtable.h>
15#include <linux/slab.h>
16#include <linux/err.h>
17#include <linux/sysctl.h>
18#include <asm/mman.h>
19#include <asm/pgalloc.h> 14#include <asm/pgalloc.h>
20#include <asm/tlb.h> 15#include <asm/tlb.h>
21#include <asm/tlbflush.h>
22#include <asm/mmu_context.h>
23#include <asm/machdep.h>
24#include <asm/cputable.h>
25#include <asm/spu.h>
26 16
27#define PAGE_SHIFT_64K 16 17#define PAGE_SHIFT_64K 16
28#define PAGE_SHIFT_16M 24 18#define PAGE_SHIFT_16M 24
29#define PAGE_SHIFT_16G 34 19#define PAGE_SHIFT_16G 34
30 20
31#define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
32#define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
33#define MAX_NUMBER_GPAGES 1024 21#define MAX_NUMBER_GPAGES 1024
34 22
35/* Tracks the 16G pages after the device tree is scanned and before the 23/* Tracks the 16G pages after the device tree is scanned and before the
@@ -502,158 +490,6 @@ unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
502 return 1UL << mmu_psize_to_shift(psize); 490 return 1UL << mmu_psize_to_shift(psize);
503} 491}
504 492
505/*
506 * Called by asm hashtable.S for doing lazy icache flush
507 */
508static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags,
509 pte_t pte, int trap, unsigned long sz)
510{
511 struct page *page;
512 int i;
513
514 if (!pfn_valid(pte_pfn(pte)))
515 return rflags;
516
517 page = pte_page(pte);
518
519 /* page is dirty */
520 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
521 if (trap == 0x400) {
522 for (i = 0; i < (sz / PAGE_SIZE); i++)
523 __flush_dcache_icache(page_address(page+i));
524 set_bit(PG_arch_1, &page->flags);
525 } else {
526 rflags |= HPTE_R_N;
527 }
528 }
529 return rflags;
530}
531
532int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
533 pte_t *ptep, unsigned long trap, int local, int ssize,
534 unsigned int shift, unsigned int mmu_psize)
535{
536 unsigned long old_pte, new_pte;
537 unsigned long va, rflags, pa, sz;
538 long slot;
539 int err = 1;
540
541 BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
542
543 /* Search the Linux page table for a match with va */
544 va = hpt_va(ea, vsid, ssize);
545
546 /*
547 * Check the user's access rights to the page. If access should be
548 * prevented then send the problem up to do_page_fault.
549 */
550 if (unlikely(access & ~pte_val(*ptep)))
551 goto out;
552 /*
553 * At this point, we have a pte (old_pte) which can be used to build
554 * or update an HPTE. There are 2 cases:
555 *
556 * 1. There is a valid (present) pte with no associated HPTE (this is
557 * the most common case)
558 * 2. There is a valid (present) pte with an associated HPTE. The
559 * current values of the pp bits in the HPTE prevent access
560 * because we are doing software DIRTY bit management and the
561 * page is currently not DIRTY.
562 */
563
564
565 do {
566 old_pte = pte_val(*ptep);
567 if (old_pte & _PAGE_BUSY)
568 goto out;
569 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
570 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
571 old_pte, new_pte));
572
573 rflags = 0x2 | (!(new_pte & _PAGE_RW));
574 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
575 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
576 sz = ((1UL) << shift);
577 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
578 /* No CPU has hugepages but lacks no execute, so we
579 * don't need to worry about that case */
580 rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte),
581 trap, sz);
582
583 /* Check if pte already has an hpte (case 2) */
584 if (unlikely(old_pte & _PAGE_HASHPTE)) {
585 /* There MIGHT be an HPTE for this pte */
586 unsigned long hash, slot;
587
588 hash = hpt_hash(va, shift, ssize);
589 if (old_pte & _PAGE_F_SECOND)
590 hash = ~hash;
591 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
592 slot += (old_pte & _PAGE_F_GIX) >> 12;
593
594 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize,
595 ssize, local) == -1)
596 old_pte &= ~_PAGE_HPTEFLAGS;
597 }
598
599 if (likely(!(old_pte & _PAGE_HASHPTE))) {
600 unsigned long hash = hpt_hash(va, shift, ssize);
601 unsigned long hpte_group;
602
603 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
604
605repeat:
606 hpte_group = ((hash & htab_hash_mask) *
607 HPTES_PER_GROUP) & ~0x7UL;
608
609 /* clear HPTE slot informations in new PTE */
610#ifdef CONFIG_PPC_64K_PAGES
611 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
612#else
613 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
614#endif
615 /* Add in WIMG bits */
616 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
617 _PAGE_COHERENT | _PAGE_GUARDED));
618
619 /* Insert into the hash table, primary slot */
620 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
621 mmu_psize, ssize);
622
623 /* Primary is full, try the secondary */
624 if (unlikely(slot == -1)) {
625 hpte_group = ((~hash & htab_hash_mask) *
626 HPTES_PER_GROUP) & ~0x7UL;
627 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
628 HPTE_V_SECONDARY,
629 mmu_psize, ssize);
630 if (slot == -1) {
631 if (mftb() & 0x1)
632 hpte_group = ((hash & htab_hash_mask) *
633 HPTES_PER_GROUP)&~0x7UL;
634
635 ppc_md.hpte_remove(hpte_group);
636 goto repeat;
637 }
638 }
639
640 if (unlikely(slot == -2))
641 panic("hash_huge_page: pte_insert failed\n");
642
643 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
644 }
645
646 /*
647 * No need to use ldarx/stdcx here
648 */
649 *ptep = __pte(new_pte & ~_PAGE_BUSY);
650
651 err = 0;
652
653 out:
654 return err;
655}
656
657static int __init add_huge_page_size(unsigned long long size) 493static int __init add_huge_page_size(unsigned long long size)
658{ 494{
659 int shift = __ffs(size); 495 int shift = __ffs(size);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-29 00:40:36 -0500