aboutsummaryrefslogtreecommitdiffstats
path: root/arch/ppc64/mm/hugetlbpage.c
diff options
context:
space:
mode:
authorDavid Gibson <david@gibson.dropbear.id.au>2005-08-05 05:39:06 -0400
committerPaul Mackerras <paulus@samba.org>2005-08-28 20:53:31 -0400
commite28f7faf05159f1cfd564596f5e6178edba6bd49 (patch)
tree45534d2c33bff8b64e3fd155fba55146cb7518e6 /arch/ppc64/mm/hugetlbpage.c
parentdecd300b30e499fe6be1bbfc5650fc971de8c1fa (diff)
[PATCH] Four level pagetables for ppc64
Implement 4-level pagetables for ppc64 This patch implements full four-level page tables for ppc64, thereby extending the usable user address range to 44 bits (16T). The patch uses a full page for the tables at the bottom and top level, and a quarter page for the intermediate levels. It uses full 64-bit pointers at every level, thus also increasing the addressable range of physical memory. This patch also tweaks the VSID allocation to allow matching range for user addresses (this halves the number of available contexts) and adds some #if and BUILD_BUG sanity checks. Signed-off-by: David Gibson <dwg@au1.ibm.com> Signed-off-by: Paul Mackerras <paulus@samba.org>
Diffstat (limited to 'arch/ppc64/mm/hugetlbpage.c')
-rw-r--r--arch/ppc64/mm/hugetlbpage.c187
1 files changed, 59 insertions, 128 deletions
diff --git a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c
index f9524602818d..a13e44230a6f 100644
--- a/arch/ppc64/mm/hugetlbpage.c
+++ b/arch/ppc64/mm/hugetlbpage.c
@@ -27,124 +27,91 @@
27 27
28#include <linux/sysctl.h> 28#include <linux/sysctl.h>
29 29
30#define HUGEPGDIR_SHIFT (HPAGE_SHIFT + PAGE_SHIFT - 3) 30/* Modelled after find_linux_pte() */
31#define HUGEPGDIR_SIZE (1UL << HUGEPGDIR_SHIFT) 31pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
32#define HUGEPGDIR_MASK (~(HUGEPGDIR_SIZE-1))
33
34#define HUGEPTE_INDEX_SIZE 9
35#define HUGEPGD_INDEX_SIZE 10
36
37#define PTRS_PER_HUGEPTE (1 << HUGEPTE_INDEX_SIZE)
38#define PTRS_PER_HUGEPGD (1 << HUGEPGD_INDEX_SIZE)
39
40static inline int hugepgd_index(unsigned long addr)
41{
42 return (addr & ~REGION_MASK) >> HUGEPGDIR_SHIFT;
43}
44
45static pud_t *hugepgd_offset(struct mm_struct *mm, unsigned long addr)
46{ 32{
47 int index; 33 pgd_t *pg;
34 pud_t *pu;
35 pmd_t *pm;
36 pte_t *pt;
48 37
49 if (! mm->context.huge_pgdir) 38 BUG_ON(! in_hugepage_area(mm->context, addr));
50 return NULL;
51 39
40 addr &= HPAGE_MASK;
41
42 pg = pgd_offset(mm, addr);
43 if (!pgd_none(*pg)) {
44 pu = pud_offset(pg, addr);
45 if (!pud_none(*pu)) {
46 pm = pmd_offset(pu, addr);
47 pt = (pte_t *)pm;
48 BUG_ON(!pmd_none(*pm)
49 && !(pte_present(*pt) && pte_huge(*pt)));
50 return pt;
51 }
52 }
52 53
53 index = hugepgd_index(addr); 54 return NULL;
54 BUG_ON(index >= PTRS_PER_HUGEPGD);
55 return (pud_t *)(mm->context.huge_pgdir + index);
56} 55}
57 56
58static inline pte_t *hugepte_offset(pud_t *dir, unsigned long addr) 57pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
59{ 58{
60 int index; 59 pgd_t *pg;
61 60 pud_t *pu;
62 if (pud_none(*dir)) 61 pmd_t *pm;
63 return NULL; 62 pte_t *pt;
64
65 index = (addr >> HPAGE_SHIFT) % PTRS_PER_HUGEPTE;
66 return (pte_t *)pud_page(*dir) + index;
67}
68 63
69static pud_t *hugepgd_alloc(struct mm_struct *mm, unsigned long addr)
70{
71 BUG_ON(! in_hugepage_area(mm->context, addr)); 64 BUG_ON(! in_hugepage_area(mm->context, addr));
72 65
73 if (! mm->context.huge_pgdir) { 66 addr &= HPAGE_MASK;
74 pgd_t *new;
75 spin_unlock(&mm->page_table_lock);
76 /* Don't use pgd_alloc(), because we want __GFP_REPEAT */
77 new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
78 BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
79 spin_lock(&mm->page_table_lock);
80
81 /*
82 * Because we dropped the lock, we should re-check the
83 * entry, as somebody else could have populated it..
84 */
85 if (mm->context.huge_pgdir)
86 pgd_free(new);
87 else
88 mm->context.huge_pgdir = new;
89 }
90 return hugepgd_offset(mm, addr);
91}
92 67
93static pte_t *hugepte_alloc(struct mm_struct *mm, pud_t *dir, unsigned long addr) 68 pg = pgd_offset(mm, addr);
94{ 69 pu = pud_alloc(mm, pg, addr);
95 if (! pud_present(*dir)) {
96 pte_t *new;
97 70
98 spin_unlock(&mm->page_table_lock); 71 if (pu) {
99 new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT); 72 pm = pmd_alloc(mm, pu, addr);
100 BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE)); 73 if (pm) {
101 spin_lock(&mm->page_table_lock); 74 pt = (pte_t *)pm;
102 /* 75 BUG_ON(!pmd_none(*pm)
103 * Because we dropped the lock, we should re-check the 76 && !(pte_present(*pt) && pte_huge(*pt)));
104 * entry, as somebody else could have populated it.. 77 return pt;
105 */
106 if (pud_present(*dir)) {
107 if (new)
108 kmem_cache_free(zero_cache, new);
109 } else {
110 struct page *ptepage;
111
112 if (! new)
113 return NULL;
114 ptepage = virt_to_page(new);
115 ptepage->mapping = (void *) mm;
116 ptepage->index = addr & HUGEPGDIR_MASK;
117 pud_populate(mm, dir, new);
118 } 78 }
119 } 79 }
120 80
121 return hugepte_offset(dir, addr); 81 return NULL;
122} 82}
123 83
124pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 84#define HUGEPTE_BATCH_SIZE (HPAGE_SIZE / PMD_SIZE)
125{
126 pud_t *pud;
127 85
128 BUG_ON(! in_hugepage_area(mm->context, addr)); 86void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
87 pte_t *ptep, pte_t pte)
88{
89 int i;
129 90
130 pud = hugepgd_offset(mm, addr); 91 if (pte_present(*ptep)) {
131 if (! pud) 92 pte_clear(mm, addr, ptep);
132 return NULL; 93 flush_tlb_pending();
94 }
133 95
134 return hugepte_offset(pud, addr); 96 for (i = 0; i < HUGEPTE_BATCH_SIZE; i++) {
97 *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
98 ptep++;
99 }
135} 100}
136 101
137pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) 102pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
103 pte_t *ptep)
138{ 104{
139 pud_t *pud; 105 unsigned long old = pte_update(ptep, ~0UL);
106 int i;
140 107
141 BUG_ON(! in_hugepage_area(mm->context, addr)); 108 if (old & _PAGE_HASHPTE)
109 hpte_update(mm, addr, old, 0);
142 110
143 pud = hugepgd_alloc(mm, addr); 111 for (i = 1; i < HUGEPTE_BATCH_SIZE; i++)
144 if (! pud) 112 ptep[i] = __pte(0);
145 return NULL;
146 113
147 return hugepte_alloc(mm, pud, addr); 114 return __pte(old);
148} 115}
149 116
150/* 117/*
@@ -541,42 +508,6 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
541 } 508 }
542} 509}
543 510
544void hugetlb_mm_free_pgd(struct mm_struct *mm)
545{
546 int i;
547 pgd_t *pgdir;
548
549 spin_lock(&mm->page_table_lock);
550
551 pgdir = mm->context.huge_pgdir;
552 if (! pgdir)
553 goto out;
554
555 mm->context.huge_pgdir = NULL;
556
557 /* cleanup any hugepte pages leftover */
558 for (i = 0; i < PTRS_PER_HUGEPGD; i++) {
559 pud_t *pud = (pud_t *)(pgdir + i);
560
561 if (! pud_none(*pud)) {
562 pte_t *pte = (pte_t *)pud_page(*pud);
563 struct page *ptepage = virt_to_page(pte);
564
565 ptepage->mapping = NULL;
566
567 BUG_ON(memcmp(pte, empty_zero_page, PAGE_SIZE));
568 kmem_cache_free(zero_cache, pte);
569 }
570 pud_clear(pud);
571 }
572
573 BUG_ON(memcmp(pgdir, empty_zero_page, PAGE_SIZE));
574 kmem_cache_free(zero_cache, pgdir);
575
576 out:
577 spin_unlock(&mm->page_table_lock);
578}
579
580int hash_huge_page(struct mm_struct *mm, unsigned long access, 511int hash_huge_page(struct mm_struct *mm, unsigned long access,
581 unsigned long ea, unsigned long vsid, int local) 512 unsigned long ea, unsigned long vsid, int local)
582{ 513{