diff options
author | David Gibson <david@gibson.dropbear.id.au> | 2005-08-05 05:39:06 -0400 |
---|---|---|
committer | Paul Mackerras <paulus@samba.org> | 2005-08-28 20:53:31 -0400 |
commit | e28f7faf05159f1cfd564596f5e6178edba6bd49 (patch) | |
tree | 45534d2c33bff8b64e3fd155fba55146cb7518e6 /arch/ppc64/mm/hugetlbpage.c | |
parent | decd300b30e499fe6be1bbfc5650fc971de8c1fa (diff) |
[PATCH] Four level pagetables for ppc64
Implement 4-level pagetables for ppc64
This patch implements full four-level page tables for ppc64, thereby
extending the usable user address range to 44 bits (16T).
The patch uses a full page for the tables at the bottom and top level,
and a quarter page for the intermediate levels. It uses full 64-bit
pointers at every level, thus also increasing the addressable range of
physical memory. This patch also tweaks the VSID allocation to allow
matching range for user addresses (this halves the number of available
contexts) and adds some #if and BUILD_BUG sanity checks.
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Signed-off-by: Paul Mackerras <paulus@samba.org>
Diffstat (limited to 'arch/ppc64/mm/hugetlbpage.c')
-rw-r--r-- | arch/ppc64/mm/hugetlbpage.c | 187 |
1 files changed, 59 insertions, 128 deletions
diff --git a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c index f9524602818d..a13e44230a6f 100644 --- a/arch/ppc64/mm/hugetlbpage.c +++ b/arch/ppc64/mm/hugetlbpage.c | |||
@@ -27,124 +27,91 @@ | |||
27 | 27 | ||
28 | #include <linux/sysctl.h> | 28 | #include <linux/sysctl.h> |
29 | 29 | ||
30 | #define HUGEPGDIR_SHIFT (HPAGE_SHIFT + PAGE_SHIFT - 3) | 30 | /* Modelled after find_linux_pte() */ |
31 | #define HUGEPGDIR_SIZE (1UL << HUGEPGDIR_SHIFT) | 31 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) |
32 | #define HUGEPGDIR_MASK (~(HUGEPGDIR_SIZE-1)) | ||
33 | |||
34 | #define HUGEPTE_INDEX_SIZE 9 | ||
35 | #define HUGEPGD_INDEX_SIZE 10 | ||
36 | |||
37 | #define PTRS_PER_HUGEPTE (1 << HUGEPTE_INDEX_SIZE) | ||
38 | #define PTRS_PER_HUGEPGD (1 << HUGEPGD_INDEX_SIZE) | ||
39 | |||
40 | static inline int hugepgd_index(unsigned long addr) | ||
41 | { | ||
42 | return (addr & ~REGION_MASK) >> HUGEPGDIR_SHIFT; | ||
43 | } | ||
44 | |||
45 | static pud_t *hugepgd_offset(struct mm_struct *mm, unsigned long addr) | ||
46 | { | 32 | { |
47 | int index; | 33 | pgd_t *pg; |
34 | pud_t *pu; | ||
35 | pmd_t *pm; | ||
36 | pte_t *pt; | ||
48 | 37 | ||
49 | if (! mm->context.huge_pgdir) | 38 | BUG_ON(! in_hugepage_area(mm->context, addr)); |
50 | return NULL; | ||
51 | 39 | ||
40 | addr &= HPAGE_MASK; | ||
41 | |||
42 | pg = pgd_offset(mm, addr); | ||
43 | if (!pgd_none(*pg)) { | ||
44 | pu = pud_offset(pg, addr); | ||
45 | if (!pud_none(*pu)) { | ||
46 | pm = pmd_offset(pu, addr); | ||
47 | pt = (pte_t *)pm; | ||
48 | BUG_ON(!pmd_none(*pm) | ||
49 | && !(pte_present(*pt) && pte_huge(*pt))); | ||
50 | return pt; | ||
51 | } | ||
52 | } | ||
52 | 53 | ||
53 | index = hugepgd_index(addr); | 54 | return NULL; |
54 | BUG_ON(index >= PTRS_PER_HUGEPGD); | ||
55 | return (pud_t *)(mm->context.huge_pgdir + index); | ||
56 | } | 55 | } |
57 | 56 | ||
58 | static inline pte_t *hugepte_offset(pud_t *dir, unsigned long addr) | 57 | pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) |
59 | { | 58 | { |
60 | int index; | 59 | pgd_t *pg; |
61 | 60 | pud_t *pu; | |
62 | if (pud_none(*dir)) | 61 | pmd_t *pm; |
63 | return NULL; | 62 | pte_t *pt; |
64 | |||
65 | index = (addr >> HPAGE_SHIFT) % PTRS_PER_HUGEPTE; | ||
66 | return (pte_t *)pud_page(*dir) + index; | ||
67 | } | ||
68 | 63 | ||
69 | static pud_t *hugepgd_alloc(struct mm_struct *mm, unsigned long addr) | ||
70 | { | ||
71 | BUG_ON(! in_hugepage_area(mm->context, addr)); | 64 | BUG_ON(! in_hugepage_area(mm->context, addr)); |
72 | 65 | ||
73 | if (! mm->context.huge_pgdir) { | 66 | addr &= HPAGE_MASK; |
74 | pgd_t *new; | ||
75 | spin_unlock(&mm->page_table_lock); | ||
76 | /* Don't use pgd_alloc(), because we want __GFP_REPEAT */ | ||
77 | new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT); | ||
78 | BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE)); | ||
79 | spin_lock(&mm->page_table_lock); | ||
80 | |||
81 | /* | ||
82 | * Because we dropped the lock, we should re-check the | ||
83 | * entry, as somebody else could have populated it.. | ||
84 | */ | ||
85 | if (mm->context.huge_pgdir) | ||
86 | pgd_free(new); | ||
87 | else | ||
88 | mm->context.huge_pgdir = new; | ||
89 | } | ||
90 | return hugepgd_offset(mm, addr); | ||
91 | } | ||
92 | 67 | ||
93 | static pte_t *hugepte_alloc(struct mm_struct *mm, pud_t *dir, unsigned long addr) | 68 | pg = pgd_offset(mm, addr); |
94 | { | 69 | pu = pud_alloc(mm, pg, addr); |
95 | if (! pud_present(*dir)) { | ||
96 | pte_t *new; | ||
97 | 70 | ||
98 | spin_unlock(&mm->page_table_lock); | 71 | if (pu) { |
99 | new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT); | 72 | pm = pmd_alloc(mm, pu, addr); |
100 | BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE)); | 73 | if (pm) { |
101 | spin_lock(&mm->page_table_lock); | 74 | pt = (pte_t *)pm; |
102 | /* | 75 | BUG_ON(!pmd_none(*pm) |
103 | * Because we dropped the lock, we should re-check the | 76 | && !(pte_present(*pt) && pte_huge(*pt))); |
104 | * entry, as somebody else could have populated it.. | 77 | return pt; |
105 | */ | ||
106 | if (pud_present(*dir)) { | ||
107 | if (new) | ||
108 | kmem_cache_free(zero_cache, new); | ||
109 | } else { | ||
110 | struct page *ptepage; | ||
111 | |||
112 | if (! new) | ||
113 | return NULL; | ||
114 | ptepage = virt_to_page(new); | ||
115 | ptepage->mapping = (void *) mm; | ||
116 | ptepage->index = addr & HUGEPGDIR_MASK; | ||
117 | pud_populate(mm, dir, new); | ||
118 | } | 78 | } |
119 | } | 79 | } |
120 | 80 | ||
121 | return hugepte_offset(dir, addr); | 81 | return NULL; |
122 | } | 82 | } |
123 | 83 | ||
124 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | 84 | #define HUGEPTE_BATCH_SIZE (HPAGE_SIZE / PMD_SIZE) |
125 | { | ||
126 | pud_t *pud; | ||
127 | 85 | ||
128 | BUG_ON(! in_hugepage_area(mm->context, addr)); | 86 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
87 | pte_t *ptep, pte_t pte) | ||
88 | { | ||
89 | int i; | ||
129 | 90 | ||
130 | pud = hugepgd_offset(mm, addr); | 91 | if (pte_present(*ptep)) { |
131 | if (! pud) | 92 | pte_clear(mm, addr, ptep); |
132 | return NULL; | 93 | flush_tlb_pending(); |
94 | } | ||
133 | 95 | ||
134 | return hugepte_offset(pud, addr); | 96 | for (i = 0; i < HUGEPTE_BATCH_SIZE; i++) { |
97 | *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); | ||
98 | ptep++; | ||
99 | } | ||
135 | } | 100 | } |
136 | 101 | ||
137 | pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) | 102 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, |
103 | pte_t *ptep) | ||
138 | { | 104 | { |
139 | pud_t *pud; | 105 | unsigned long old = pte_update(ptep, ~0UL); |
106 | int i; | ||
140 | 107 | ||
141 | BUG_ON(! in_hugepage_area(mm->context, addr)); | 108 | if (old & _PAGE_HASHPTE) |
109 | hpte_update(mm, addr, old, 0); | ||
142 | 110 | ||
143 | pud = hugepgd_alloc(mm, addr); | 111 | for (i = 1; i < HUGEPTE_BATCH_SIZE; i++) |
144 | if (! pud) | 112 | ptep[i] = __pte(0); |
145 | return NULL; | ||
146 | 113 | ||
147 | return hugepte_alloc(mm, pud, addr); | 114 | return __pte(old); |
148 | } | 115 | } |
149 | 116 | ||
150 | /* | 117 | /* |
@@ -541,42 +508,6 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, | |||
541 | } | 508 | } |
542 | } | 509 | } |
543 | 510 | ||
544 | void hugetlb_mm_free_pgd(struct mm_struct *mm) | ||
545 | { | ||
546 | int i; | ||
547 | pgd_t *pgdir; | ||
548 | |||
549 | spin_lock(&mm->page_table_lock); | ||
550 | |||
551 | pgdir = mm->context.huge_pgdir; | ||
552 | if (! pgdir) | ||
553 | goto out; | ||
554 | |||
555 | mm->context.huge_pgdir = NULL; | ||
556 | |||
557 | /* cleanup any hugepte pages leftover */ | ||
558 | for (i = 0; i < PTRS_PER_HUGEPGD; i++) { | ||
559 | pud_t *pud = (pud_t *)(pgdir + i); | ||
560 | |||
561 | if (! pud_none(*pud)) { | ||
562 | pte_t *pte = (pte_t *)pud_page(*pud); | ||
563 | struct page *ptepage = virt_to_page(pte); | ||
564 | |||
565 | ptepage->mapping = NULL; | ||
566 | |||
567 | BUG_ON(memcmp(pte, empty_zero_page, PAGE_SIZE)); | ||
568 | kmem_cache_free(zero_cache, pte); | ||
569 | } | ||
570 | pud_clear(pud); | ||
571 | } | ||
572 | |||
573 | BUG_ON(memcmp(pgdir, empty_zero_page, PAGE_SIZE)); | ||
574 | kmem_cache_free(zero_cache, pgdir); | ||
575 | |||
576 | out: | ||
577 | spin_unlock(&mm->page_table_lock); | ||
578 | } | ||
579 | |||
580 | int hash_huge_page(struct mm_struct *mm, unsigned long access, | 511 | int hash_huge_page(struct mm_struct *mm, unsigned long access, |
581 | unsigned long ea, unsigned long vsid, int local) | 512 | unsigned long ea, unsigned long vsid, int local) |
582 | { | 513 | { |