diff options
author | Jon Tollefson <kniht@linux.vnet.ibm.com> | 2008-07-24 00:27:56 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2008-07-24 13:47:19 -0400 |
commit | 0d9ea75443dc7e37843e656b8ebc947a6d16d618 (patch) | |
tree | 4eb47a4468a92c90a90873d1646196955b487996 /arch/powerpc/mm/hugetlbpage.c | |
parent | f4a67cceee4a6f5ed38011a698c9e34747270ae5 (diff) |
powerpc: support multiple hugepage sizes
Instead of using the variable mmu_huge_psize to keep track of the huge
page size we use an array of MMU_PAGE_* values. For each supported huge
page size we need to know the hugepte_shift value and have a
pgtable_cache. The hstate or an mmu_huge_psizes index is passed to
functions so that they know which huge page size they should use.
The hugepage sizes 16M and 64K are setup(if available on the hardware) so
that they don't have to be set on the boot cmd line in order to use them.
The number of 16G pages have to be specified at boot-time though (e.g.
hugepagesz=16G hugepages=5).
Signed-off-by: Jon Tollefson <kniht@linux.vnet.ibm.com>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'arch/powerpc/mm/hugetlbpage.c')
-rw-r--r-- | arch/powerpc/mm/hugetlbpage.c | 274 |
1 files changed, 177 insertions, 97 deletions
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 19b1a9cec6d5..fb42c4dd3217 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c | |||
@@ -37,15 +37,30 @@ | |||
37 | static unsigned long gpage_freearray[MAX_NUMBER_GPAGES]; | 37 | static unsigned long gpage_freearray[MAX_NUMBER_GPAGES]; |
38 | static unsigned nr_gpages; | 38 | static unsigned nr_gpages; |
39 | 39 | ||
40 | unsigned int hugepte_shift; | 40 | /* Array of valid huge page sizes - non-zero value(hugepte_shift) is |
41 | #define PTRS_PER_HUGEPTE (1 << hugepte_shift) | 41 | * stored for the huge page sizes that are valid. |
42 | #define HUGEPTE_TABLE_SIZE (sizeof(pte_t) << hugepte_shift) | 42 | */ |
43 | unsigned int mmu_huge_psizes[MMU_PAGE_COUNT] = { }; /* initialize all to 0 */ | ||
44 | |||
45 | #define hugepte_shift mmu_huge_psizes | ||
46 | #define PTRS_PER_HUGEPTE(psize) (1 << hugepte_shift[psize]) | ||
47 | #define HUGEPTE_TABLE_SIZE(psize) (sizeof(pte_t) << hugepte_shift[psize]) | ||
48 | |||
49 | #define HUGEPD_SHIFT(psize) (mmu_psize_to_shift(psize) \ | ||
50 | + hugepte_shift[psize]) | ||
51 | #define HUGEPD_SIZE(psize) (1UL << HUGEPD_SHIFT(psize)) | ||
52 | #define HUGEPD_MASK(psize) (~(HUGEPD_SIZE(psize)-1)) | ||
43 | 53 | ||
44 | #define HUGEPD_SHIFT (HPAGE_SHIFT + hugepte_shift) | 54 | /* Subtract one from array size because we don't need a cache for 4K since |
45 | #define HUGEPD_SIZE (1UL << HUGEPD_SHIFT) | 55 | * is not a huge page size */ |
46 | #define HUGEPD_MASK (~(HUGEPD_SIZE-1)) | 56 | #define huge_pgtable_cache(psize) (pgtable_cache[HUGEPTE_CACHE_NUM \ |
57 | + psize-1]) | ||
58 | #define HUGEPTE_CACHE_NAME(psize) (huge_pgtable_cache_name[psize]) | ||
47 | 59 | ||
48 | #define huge_pgtable_cache (pgtable_cache[HUGEPTE_CACHE_NUM]) | 60 | static const char *huge_pgtable_cache_name[MMU_PAGE_COUNT] = { |
61 | "unused_4K", "hugepte_cache_64K", "unused_64K_AP", | ||
62 | "hugepte_cache_1M", "hugepte_cache_16M", "hugepte_cache_16G" | ||
63 | }; | ||
49 | 64 | ||
50 | /* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad() | 65 | /* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad() |
51 | * will choke on pointers to hugepte tables, which is handy for | 66 | * will choke on pointers to hugepte tables, which is handy for |
@@ -56,24 +71,49 @@ typedef struct { unsigned long pd; } hugepd_t; | |||
56 | 71 | ||
57 | #define hugepd_none(hpd) ((hpd).pd == 0) | 72 | #define hugepd_none(hpd) ((hpd).pd == 0) |
58 | 73 | ||
74 | static inline int shift_to_mmu_psize(unsigned int shift) | ||
75 | { | ||
76 | switch (shift) { | ||
77 | #ifndef CONFIG_PPC_64K_PAGES | ||
78 | case PAGE_SHIFT_64K: | ||
79 | return MMU_PAGE_64K; | ||
80 | #endif | ||
81 | case PAGE_SHIFT_16M: | ||
82 | return MMU_PAGE_16M; | ||
83 | case PAGE_SHIFT_16G: | ||
84 | return MMU_PAGE_16G; | ||
85 | } | ||
86 | return -1; | ||
87 | } | ||
88 | |||
89 | static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) | ||
90 | { | ||
91 | if (mmu_psize_defs[mmu_psize].shift) | ||
92 | return mmu_psize_defs[mmu_psize].shift; | ||
93 | BUG(); | ||
94 | } | ||
95 | |||
59 | static inline pte_t *hugepd_page(hugepd_t hpd) | 96 | static inline pte_t *hugepd_page(hugepd_t hpd) |
60 | { | 97 | { |
61 | BUG_ON(!(hpd.pd & HUGEPD_OK)); | 98 | BUG_ON(!(hpd.pd & HUGEPD_OK)); |
62 | return (pte_t *)(hpd.pd & ~HUGEPD_OK); | 99 | return (pte_t *)(hpd.pd & ~HUGEPD_OK); |
63 | } | 100 | } |
64 | 101 | ||
65 | static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr) | 102 | static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr, |
103 | struct hstate *hstate) | ||
66 | { | 104 | { |
67 | unsigned long idx = ((addr >> HPAGE_SHIFT) & (PTRS_PER_HUGEPTE-1)); | 105 | unsigned int shift = huge_page_shift(hstate); |
106 | int psize = shift_to_mmu_psize(shift); | ||
107 | unsigned long idx = ((addr >> shift) & (PTRS_PER_HUGEPTE(psize)-1)); | ||
68 | pte_t *dir = hugepd_page(*hpdp); | 108 | pte_t *dir = hugepd_page(*hpdp); |
69 | 109 | ||
70 | return dir + idx; | 110 | return dir + idx; |
71 | } | 111 | } |
72 | 112 | ||
73 | static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, | 113 | static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, |
74 | unsigned long address) | 114 | unsigned long address, unsigned int psize) |
75 | { | 115 | { |
76 | pte_t *new = kmem_cache_alloc(huge_pgtable_cache, | 116 | pte_t *new = kmem_cache_alloc(huge_pgtable_cache(psize), |
77 | GFP_KERNEL|__GFP_REPEAT); | 117 | GFP_KERNEL|__GFP_REPEAT); |
78 | 118 | ||
79 | if (! new) | 119 | if (! new) |
@@ -81,7 +121,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, | |||
81 | 121 | ||
82 | spin_lock(&mm->page_table_lock); | 122 | spin_lock(&mm->page_table_lock); |
83 | if (!hugepd_none(*hpdp)) | 123 | if (!hugepd_none(*hpdp)) |
84 | kmem_cache_free(huge_pgtable_cache, new); | 124 | kmem_cache_free(huge_pgtable_cache(psize), new); |
85 | else | 125 | else |
86 | hpdp->pd = (unsigned long)new | HUGEPD_OK; | 126 | hpdp->pd = (unsigned long)new | HUGEPD_OK; |
87 | spin_unlock(&mm->page_table_lock); | 127 | spin_unlock(&mm->page_table_lock); |
@@ -90,21 +130,22 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, | |||
90 | 130 | ||
91 | /* Base page size affects how we walk hugetlb page tables */ | 131 | /* Base page size affects how we walk hugetlb page tables */ |
92 | #ifdef CONFIG_PPC_64K_PAGES | 132 | #ifdef CONFIG_PPC_64K_PAGES |
93 | #define hpmd_offset(pud, addr) pmd_offset(pud, addr) | 133 | #define hpmd_offset(pud, addr, h) pmd_offset(pud, addr) |
94 | #define hpmd_alloc(mm, pud, addr) pmd_alloc(mm, pud, addr) | 134 | #define hpmd_alloc(mm, pud, addr, h) pmd_alloc(mm, pud, addr) |
95 | #else | 135 | #else |
96 | static inline | 136 | static inline |
97 | pmd_t *hpmd_offset(pud_t *pud, unsigned long addr) | 137 | pmd_t *hpmd_offset(pud_t *pud, unsigned long addr, struct hstate *hstate) |
98 | { | 138 | { |
99 | if (HPAGE_SHIFT == PAGE_SHIFT_64K) | 139 | if (huge_page_shift(hstate) == PAGE_SHIFT_64K) |
100 | return pmd_offset(pud, addr); | 140 | return pmd_offset(pud, addr); |
101 | else | 141 | else |
102 | return (pmd_t *) pud; | 142 | return (pmd_t *) pud; |
103 | } | 143 | } |
104 | static inline | 144 | static inline |
105 | pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr) | 145 | pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr, |
146 | struct hstate *hstate) | ||
106 | { | 147 | { |
107 | if (HPAGE_SHIFT == PAGE_SHIFT_64K) | 148 | if (huge_page_shift(hstate) == PAGE_SHIFT_64K) |
108 | return pmd_alloc(mm, pud, addr); | 149 | return pmd_alloc(mm, pud, addr); |
109 | else | 150 | else |
110 | return (pmd_t *) pud; | 151 | return (pmd_t *) pud; |
@@ -128,8 +169,9 @@ void add_gpage(unsigned long addr, unsigned long page_size, | |||
128 | } | 169 | } |
129 | 170 | ||
130 | /* Moves the gigantic page addresses from the temporary list to the | 171 | /* Moves the gigantic page addresses from the temporary list to the |
131 | * huge_boot_pages list. */ | 172 | * huge_boot_pages list. |
132 | int alloc_bootmem_huge_page(struct hstate *h) | 173 | */ |
174 | int alloc_bootmem_huge_page(struct hstate *hstate) | ||
133 | { | 175 | { |
134 | struct huge_bootmem_page *m; | 176 | struct huge_bootmem_page *m; |
135 | if (nr_gpages == 0) | 177 | if (nr_gpages == 0) |
@@ -137,7 +179,7 @@ int alloc_bootmem_huge_page(struct hstate *h) | |||
137 | m = phys_to_virt(gpage_freearray[--nr_gpages]); | 179 | m = phys_to_virt(gpage_freearray[--nr_gpages]); |
138 | gpage_freearray[nr_gpages] = 0; | 180 | gpage_freearray[nr_gpages] = 0; |
139 | list_add(&m->list, &huge_boot_pages); | 181 | list_add(&m->list, &huge_boot_pages); |
140 | m->hstate = h; | 182 | m->hstate = hstate; |
141 | return 1; | 183 | return 1; |
142 | } | 184 | } |
143 | 185 | ||
@@ -149,17 +191,25 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | |||
149 | pud_t *pu; | 191 | pud_t *pu; |
150 | pmd_t *pm; | 192 | pmd_t *pm; |
151 | 193 | ||
152 | BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize); | 194 | unsigned int psize; |
195 | unsigned int shift; | ||
196 | unsigned long sz; | ||
197 | struct hstate *hstate; | ||
198 | psize = get_slice_psize(mm, addr); | ||
199 | shift = mmu_psize_to_shift(psize); | ||
200 | sz = ((1UL) << shift); | ||
201 | hstate = size_to_hstate(sz); | ||
153 | 202 | ||
154 | addr &= HPAGE_MASK; | 203 | addr &= hstate->mask; |
155 | 204 | ||
156 | pg = pgd_offset(mm, addr); | 205 | pg = pgd_offset(mm, addr); |
157 | if (!pgd_none(*pg)) { | 206 | if (!pgd_none(*pg)) { |
158 | pu = pud_offset(pg, addr); | 207 | pu = pud_offset(pg, addr); |
159 | if (!pud_none(*pu)) { | 208 | if (!pud_none(*pu)) { |
160 | pm = hpmd_offset(pu, addr); | 209 | pm = hpmd_offset(pu, addr, hstate); |
161 | if (!pmd_none(*pm)) | 210 | if (!pmd_none(*pm)) |
162 | return hugepte_offset((hugepd_t *)pm, addr); | 211 | return hugepte_offset((hugepd_t *)pm, addr, |
212 | hstate); | ||
163 | } | 213 | } |
164 | } | 214 | } |
165 | 215 | ||
@@ -173,16 +223,20 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, | |||
173 | pud_t *pu; | 223 | pud_t *pu; |
174 | pmd_t *pm; | 224 | pmd_t *pm; |
175 | hugepd_t *hpdp = NULL; | 225 | hugepd_t *hpdp = NULL; |
226 | struct hstate *hstate; | ||
227 | unsigned int psize; | ||
228 | hstate = size_to_hstate(sz); | ||
176 | 229 | ||
177 | BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize); | 230 | psize = get_slice_psize(mm, addr); |
231 | BUG_ON(!mmu_huge_psizes[psize]); | ||
178 | 232 | ||
179 | addr &= HPAGE_MASK; | 233 | addr &= hstate->mask; |
180 | 234 | ||
181 | pg = pgd_offset(mm, addr); | 235 | pg = pgd_offset(mm, addr); |
182 | pu = pud_alloc(mm, pg, addr); | 236 | pu = pud_alloc(mm, pg, addr); |
183 | 237 | ||
184 | if (pu) { | 238 | if (pu) { |
185 | pm = hpmd_alloc(mm, pu, addr); | 239 | pm = hpmd_alloc(mm, pu, addr, hstate); |
186 | if (pm) | 240 | if (pm) |
187 | hpdp = (hugepd_t *)pm; | 241 | hpdp = (hugepd_t *)pm; |
188 | } | 242 | } |
@@ -190,10 +244,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, | |||
190 | if (! hpdp) | 244 | if (! hpdp) |
191 | return NULL; | 245 | return NULL; |
192 | 246 | ||
193 | if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr)) | 247 | if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, psize)) |
194 | return NULL; | 248 | return NULL; |
195 | 249 | ||
196 | return hugepte_offset(hpdp, addr); | 250 | return hugepte_offset(hpdp, addr, hstate); |
197 | } | 251 | } |
198 | 252 | ||
199 | int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) | 253 | int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) |
@@ -201,19 +255,22 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) | |||
201 | return 0; | 255 | return 0; |
202 | } | 256 | } |
203 | 257 | ||
204 | static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp) | 258 | static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp, |
259 | unsigned int psize) | ||
205 | { | 260 | { |
206 | pte_t *hugepte = hugepd_page(*hpdp); | 261 | pte_t *hugepte = hugepd_page(*hpdp); |
207 | 262 | ||
208 | hpdp->pd = 0; | 263 | hpdp->pd = 0; |
209 | tlb->need_flush = 1; | 264 | tlb->need_flush = 1; |
210 | pgtable_free_tlb(tlb, pgtable_free_cache(hugepte, HUGEPTE_CACHE_NUM, | 265 | pgtable_free_tlb(tlb, pgtable_free_cache(hugepte, |
266 | HUGEPTE_CACHE_NUM+psize-1, | ||
211 | PGF_CACHENUM_MASK)); | 267 | PGF_CACHENUM_MASK)); |
212 | } | 268 | } |
213 | 269 | ||
214 | static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, | 270 | static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, |
215 | unsigned long addr, unsigned long end, | 271 | unsigned long addr, unsigned long end, |
216 | unsigned long floor, unsigned long ceiling) | 272 | unsigned long floor, unsigned long ceiling, |
273 | unsigned int psize) | ||
217 | { | 274 | { |
218 | pmd_t *pmd; | 275 | pmd_t *pmd; |
219 | unsigned long next; | 276 | unsigned long next; |
@@ -225,7 +282,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, | |||
225 | next = pmd_addr_end(addr, end); | 282 | next = pmd_addr_end(addr, end); |
226 | if (pmd_none(*pmd)) | 283 | if (pmd_none(*pmd)) |
227 | continue; | 284 | continue; |
228 | free_hugepte_range(tlb, (hugepd_t *)pmd); | 285 | free_hugepte_range(tlb, (hugepd_t *)pmd, psize); |
229 | } while (pmd++, addr = next, addr != end); | 286 | } while (pmd++, addr = next, addr != end); |
230 | 287 | ||
231 | start &= PUD_MASK; | 288 | start &= PUD_MASK; |
@@ -251,6 +308,9 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, | |||
251 | pud_t *pud; | 308 | pud_t *pud; |
252 | unsigned long next; | 309 | unsigned long next; |
253 | unsigned long start; | 310 | unsigned long start; |
311 | unsigned int shift; | ||
312 | unsigned int psize = get_slice_psize(tlb->mm, addr); | ||
313 | shift = mmu_psize_to_shift(psize); | ||
254 | 314 | ||
255 | start = addr; | 315 | start = addr; |
256 | pud = pud_offset(pgd, addr); | 316 | pud = pud_offset(pgd, addr); |
@@ -259,16 +319,18 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, | |||
259 | #ifdef CONFIG_PPC_64K_PAGES | 319 | #ifdef CONFIG_PPC_64K_PAGES |
260 | if (pud_none_or_clear_bad(pud)) | 320 | if (pud_none_or_clear_bad(pud)) |
261 | continue; | 321 | continue; |
262 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling); | 322 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling, |
323 | psize); | ||
263 | #else | 324 | #else |
264 | if (HPAGE_SHIFT == PAGE_SHIFT_64K) { | 325 | if (shift == PAGE_SHIFT_64K) { |
265 | if (pud_none_or_clear_bad(pud)) | 326 | if (pud_none_or_clear_bad(pud)) |
266 | continue; | 327 | continue; |
267 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling); | 328 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, |
329 | ceiling, psize); | ||
268 | } else { | 330 | } else { |
269 | if (pud_none(*pud)) | 331 | if (pud_none(*pud)) |
270 | continue; | 332 | continue; |
271 | free_hugepte_range(tlb, (hugepd_t *)pud); | 333 | free_hugepte_range(tlb, (hugepd_t *)pud, psize); |
272 | } | 334 | } |
273 | #endif | 335 | #endif |
274 | } while (pud++, addr = next, addr != end); | 336 | } while (pud++, addr = next, addr != end); |
@@ -336,27 +398,29 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, | |||
336 | * now has no other vmas using it, so can be freed, we don't | 398 | * now has no other vmas using it, so can be freed, we don't |
337 | * bother to round floor or end up - the tests don't need that. | 399 | * bother to round floor or end up - the tests don't need that. |
338 | */ | 400 | */ |
401 | unsigned int psize = get_slice_psize(tlb->mm, addr); | ||
339 | 402 | ||
340 | addr &= HUGEPD_MASK; | 403 | addr &= HUGEPD_MASK(psize); |
341 | if (addr < floor) { | 404 | if (addr < floor) { |
342 | addr += HUGEPD_SIZE; | 405 | addr += HUGEPD_SIZE(psize); |
343 | if (!addr) | 406 | if (!addr) |
344 | return; | 407 | return; |
345 | } | 408 | } |
346 | if (ceiling) { | 409 | if (ceiling) { |
347 | ceiling &= HUGEPD_MASK; | 410 | ceiling &= HUGEPD_MASK(psize); |
348 | if (!ceiling) | 411 | if (!ceiling) |
349 | return; | 412 | return; |
350 | } | 413 | } |
351 | if (end - 1 > ceiling - 1) | 414 | if (end - 1 > ceiling - 1) |
352 | end -= HUGEPD_SIZE; | 415 | end -= HUGEPD_SIZE(psize); |
353 | if (addr > end - 1) | 416 | if (addr > end - 1) |
354 | return; | 417 | return; |
355 | 418 | ||
356 | start = addr; | 419 | start = addr; |
357 | pgd = pgd_offset(tlb->mm, addr); | 420 | pgd = pgd_offset(tlb->mm, addr); |
358 | do { | 421 | do { |
359 | BUG_ON(get_slice_psize(tlb->mm, addr) != mmu_huge_psize); | 422 | psize = get_slice_psize(tlb->mm, addr); |
423 | BUG_ON(!mmu_huge_psizes[psize]); | ||
360 | next = pgd_addr_end(addr, end); | 424 | next = pgd_addr_end(addr, end); |
361 | if (pgd_none_or_clear_bad(pgd)) | 425 | if (pgd_none_or_clear_bad(pgd)) |
362 | continue; | 426 | continue; |
@@ -373,7 +437,11 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, | |||
373 | * necessary anymore if we make hpte_need_flush() get the | 437 | * necessary anymore if we make hpte_need_flush() get the |
374 | * page size from the slices | 438 | * page size from the slices |
375 | */ | 439 | */ |
376 | pte_update(mm, addr & HPAGE_MASK, ptep, ~0UL, 1); | 440 | unsigned int psize = get_slice_psize(mm, addr); |
441 | unsigned int shift = mmu_psize_to_shift(psize); | ||
442 | unsigned long sz = ((1UL) << shift); | ||
443 | struct hstate *hstate = size_to_hstate(sz); | ||
444 | pte_update(mm, addr & hstate->mask, ptep, ~0UL, 1); | ||
377 | } | 445 | } |
378 | *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); | 446 | *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); |
379 | } | 447 | } |
@@ -390,14 +458,19 @@ follow_huge_addr(struct mm_struct *mm, unsigned long address, int write) | |||
390 | { | 458 | { |
391 | pte_t *ptep; | 459 | pte_t *ptep; |
392 | struct page *page; | 460 | struct page *page; |
461 | unsigned int mmu_psize = get_slice_psize(mm, address); | ||
393 | 462 | ||
394 | if (get_slice_psize(mm, address) != mmu_huge_psize) | 463 | /* Verify it is a huge page else bail. */ |
464 | if (!mmu_huge_psizes[mmu_psize]) | ||
395 | return ERR_PTR(-EINVAL); | 465 | return ERR_PTR(-EINVAL); |
396 | 466 | ||
397 | ptep = huge_pte_offset(mm, address); | 467 | ptep = huge_pte_offset(mm, address); |
398 | page = pte_page(*ptep); | 468 | page = pte_page(*ptep); |
399 | if (page) | 469 | if (page) { |
400 | page += (address % HPAGE_SIZE) / PAGE_SIZE; | 470 | unsigned int shift = mmu_psize_to_shift(mmu_psize); |
471 | unsigned long sz = ((1UL) << shift); | ||
472 | page += (address % sz) / PAGE_SIZE; | ||
473 | } | ||
401 | 474 | ||
402 | return page; | 475 | return page; |
403 | } | 476 | } |
@@ -425,15 +498,16 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, | |||
425 | unsigned long len, unsigned long pgoff, | 498 | unsigned long len, unsigned long pgoff, |
426 | unsigned long flags) | 499 | unsigned long flags) |
427 | { | 500 | { |
428 | return slice_get_unmapped_area(addr, len, flags, | 501 | struct hstate *hstate = hstate_file(file); |
429 | mmu_huge_psize, 1, 0); | 502 | int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); |
503 | return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0); | ||
430 | } | 504 | } |
431 | 505 | ||
432 | /* | 506 | /* |
433 | * Called by asm hashtable.S for doing lazy icache flush | 507 | * Called by asm hashtable.S for doing lazy icache flush |
434 | */ | 508 | */ |
435 | static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags, | 509 | static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags, |
436 | pte_t pte, int trap) | 510 | pte_t pte, int trap, unsigned long sz) |
437 | { | 511 | { |
438 | struct page *page; | 512 | struct page *page; |
439 | int i; | 513 | int i; |
@@ -446,7 +520,7 @@ static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags, | |||
446 | /* page is dirty */ | 520 | /* page is dirty */ |
447 | if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) { | 521 | if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) { |
448 | if (trap == 0x400) { | 522 | if (trap == 0x400) { |
449 | for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) | 523 | for (i = 0; i < (sz / PAGE_SIZE); i++) |
450 | __flush_dcache_icache(page_address(page+i)); | 524 | __flush_dcache_icache(page_address(page+i)); |
451 | set_bit(PG_arch_1, &page->flags); | 525 | set_bit(PG_arch_1, &page->flags); |
452 | } else { | 526 | } else { |
@@ -462,11 +536,16 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access, | |||
462 | { | 536 | { |
463 | pte_t *ptep; | 537 | pte_t *ptep; |
464 | unsigned long old_pte, new_pte; | 538 | unsigned long old_pte, new_pte; |
465 | unsigned long va, rflags, pa; | 539 | unsigned long va, rflags, pa, sz; |
466 | long slot; | 540 | long slot; |
467 | int err = 1; | 541 | int err = 1; |
468 | int ssize = user_segment_size(ea); | 542 | int ssize = user_segment_size(ea); |
543 | unsigned int mmu_psize; | ||
544 | int shift; | ||
545 | mmu_psize = get_slice_psize(mm, ea); | ||
469 | 546 | ||
547 | if (!mmu_huge_psizes[mmu_psize]) | ||
548 | goto out; | ||
470 | ptep = huge_pte_offset(mm, ea); | 549 | ptep = huge_pte_offset(mm, ea); |
471 | 550 | ||
472 | /* Search the Linux page table for a match with va */ | 551 | /* Search the Linux page table for a match with va */ |
@@ -509,30 +588,32 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access, | |||
509 | rflags = 0x2 | (!(new_pte & _PAGE_RW)); | 588 | rflags = 0x2 | (!(new_pte & _PAGE_RW)); |
510 | /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */ | 589 | /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */ |
511 | rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N); | 590 | rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N); |
591 | shift = mmu_psize_to_shift(mmu_psize); | ||
592 | sz = ((1UL) << shift); | ||
512 | if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) | 593 | if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) |
513 | /* No CPU has hugepages but lacks no execute, so we | 594 | /* No CPU has hugepages but lacks no execute, so we |
514 | * don't need to worry about that case */ | 595 | * don't need to worry about that case */ |
515 | rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte), | 596 | rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte), |
516 | trap); | 597 | trap, sz); |
517 | 598 | ||
518 | /* Check if pte already has an hpte (case 2) */ | 599 | /* Check if pte already has an hpte (case 2) */ |
519 | if (unlikely(old_pte & _PAGE_HASHPTE)) { | 600 | if (unlikely(old_pte & _PAGE_HASHPTE)) { |
520 | /* There MIGHT be an HPTE for this pte */ | 601 | /* There MIGHT be an HPTE for this pte */ |
521 | unsigned long hash, slot; | 602 | unsigned long hash, slot; |
522 | 603 | ||
523 | hash = hpt_hash(va, HPAGE_SHIFT, ssize); | 604 | hash = hpt_hash(va, shift, ssize); |
524 | if (old_pte & _PAGE_F_SECOND) | 605 | if (old_pte & _PAGE_F_SECOND) |
525 | hash = ~hash; | 606 | hash = ~hash; |
526 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | 607 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; |
527 | slot += (old_pte & _PAGE_F_GIX) >> 12; | 608 | slot += (old_pte & _PAGE_F_GIX) >> 12; |
528 | 609 | ||
529 | if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize, | 610 | if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize, |
530 | ssize, local) == -1) | 611 | ssize, local) == -1) |
531 | old_pte &= ~_PAGE_HPTEFLAGS; | 612 | old_pte &= ~_PAGE_HPTEFLAGS; |
532 | } | 613 | } |
533 | 614 | ||
534 | if (likely(!(old_pte & _PAGE_HASHPTE))) { | 615 | if (likely(!(old_pte & _PAGE_HASHPTE))) { |
535 | unsigned long hash = hpt_hash(va, HPAGE_SHIFT, ssize); | 616 | unsigned long hash = hpt_hash(va, shift, ssize); |
536 | unsigned long hpte_group; | 617 | unsigned long hpte_group; |
537 | 618 | ||
538 | pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; | 619 | pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; |
@@ -553,7 +634,7 @@ repeat: | |||
553 | 634 | ||
554 | /* Insert into the hash table, primary slot */ | 635 | /* Insert into the hash table, primary slot */ |
555 | slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, | 636 | slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, |
556 | mmu_huge_psize, ssize); | 637 | mmu_psize, ssize); |
557 | 638 | ||
558 | /* Primary is full, try the secondary */ | 639 | /* Primary is full, try the secondary */ |
559 | if (unlikely(slot == -1)) { | 640 | if (unlikely(slot == -1)) { |
@@ -561,7 +642,7 @@ repeat: | |||
561 | HPTES_PER_GROUP) & ~0x7UL; | 642 | HPTES_PER_GROUP) & ~0x7UL; |
562 | slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, | 643 | slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, |
563 | HPTE_V_SECONDARY, | 644 | HPTE_V_SECONDARY, |
564 | mmu_huge_psize, ssize); | 645 | mmu_psize, ssize); |
565 | if (slot == -1) { | 646 | if (slot == -1) { |
566 | if (mftb() & 0x1) | 647 | if (mftb() & 0x1) |
567 | hpte_group = ((hash & htab_hash_mask) * | 648 | hpte_group = ((hash & htab_hash_mask) * |
@@ -598,66 +679,50 @@ void set_huge_psize(int psize) | |||
598 | (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT || | 679 | (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT || |
599 | mmu_psize_defs[psize].shift == PAGE_SHIFT_64K || | 680 | mmu_psize_defs[psize].shift == PAGE_SHIFT_64K || |
600 | mmu_psize_defs[psize].shift == PAGE_SHIFT_16G)) { | 681 | mmu_psize_defs[psize].shift == PAGE_SHIFT_16G)) { |
601 | /* Return if huge page size is the same as the | 682 | /* Return if huge page size has already been setup or is the |
602 | * base page size. */ | 683 | * same as the base page size. */ |
603 | if (mmu_psize_defs[psize].shift == PAGE_SHIFT) | 684 | if (mmu_huge_psizes[psize] || |
685 | mmu_psize_defs[psize].shift == PAGE_SHIFT) | ||
604 | return; | 686 | return; |
687 | hugetlb_add_hstate(mmu_psize_defs[psize].shift - PAGE_SHIFT); | ||
605 | 688 | ||
606 | HPAGE_SHIFT = mmu_psize_defs[psize].shift; | 689 | switch (mmu_psize_defs[psize].shift) { |
607 | mmu_huge_psize = psize; | ||
608 | |||
609 | switch (HPAGE_SHIFT) { | ||
610 | case PAGE_SHIFT_64K: | 690 | case PAGE_SHIFT_64K: |
611 | /* We only allow 64k hpages with 4k base page, | 691 | /* We only allow 64k hpages with 4k base page, |
612 | * which was checked above, and always put them | 692 | * which was checked above, and always put them |
613 | * at the PMD */ | 693 | * at the PMD */ |
614 | hugepte_shift = PMD_SHIFT; | 694 | hugepte_shift[psize] = PMD_SHIFT; |
615 | break; | 695 | break; |
616 | case PAGE_SHIFT_16M: | 696 | case PAGE_SHIFT_16M: |
617 | /* 16M pages can be at two different levels | 697 | /* 16M pages can be at two different levels |
618 | * of pagestables based on base page size */ | 698 | * of pagestables based on base page size */ |
619 | if (PAGE_SHIFT == PAGE_SHIFT_64K) | 699 | if (PAGE_SHIFT == PAGE_SHIFT_64K) |
620 | hugepte_shift = PMD_SHIFT; | 700 | hugepte_shift[psize] = PMD_SHIFT; |
621 | else /* 4k base page */ | 701 | else /* 4k base page */ |
622 | hugepte_shift = PUD_SHIFT; | 702 | hugepte_shift[psize] = PUD_SHIFT; |
623 | break; | 703 | break; |
624 | case PAGE_SHIFT_16G: | 704 | case PAGE_SHIFT_16G: |
625 | /* 16G pages are always at PGD level */ | 705 | /* 16G pages are always at PGD level */ |
626 | hugepte_shift = PGDIR_SHIFT; | 706 | hugepte_shift[psize] = PGDIR_SHIFT; |
627 | break; | 707 | break; |
628 | } | 708 | } |
629 | hugepte_shift -= HPAGE_SHIFT; | 709 | hugepte_shift[psize] -= mmu_psize_defs[psize].shift; |
630 | } else | 710 | } else |
631 | HPAGE_SHIFT = 0; | 711 | hugepte_shift[psize] = 0; |
632 | } | 712 | } |
633 | 713 | ||
634 | static int __init hugepage_setup_sz(char *str) | 714 | static int __init hugepage_setup_sz(char *str) |
635 | { | 715 | { |
636 | unsigned long long size; | 716 | unsigned long long size; |
637 | int mmu_psize = -1; | 717 | int mmu_psize; |
638 | int shift; | 718 | int shift; |
639 | 719 | ||
640 | size = memparse(str, &str); | 720 | size = memparse(str, &str); |
641 | 721 | ||
642 | shift = __ffs(size); | 722 | shift = __ffs(size); |
643 | switch (shift) { | 723 | mmu_psize = shift_to_mmu_psize(shift); |
644 | #ifndef CONFIG_PPC_64K_PAGES | 724 | if (mmu_psize >= 0 && mmu_psize_defs[mmu_psize].shift) |
645 | case PAGE_SHIFT_64K: | ||
646 | mmu_psize = MMU_PAGE_64K; | ||
647 | break; | ||
648 | #endif | ||
649 | case PAGE_SHIFT_16M: | ||
650 | mmu_psize = MMU_PAGE_16M; | ||
651 | break; | ||
652 | case PAGE_SHIFT_16G: | ||
653 | mmu_psize = MMU_PAGE_16G; | ||
654 | break; | ||
655 | } | ||
656 | |||
657 | if (mmu_psize >= 0 && mmu_psize_defs[mmu_psize].shift) { | ||
658 | set_huge_psize(mmu_psize); | 725 | set_huge_psize(mmu_psize); |
659 | hugetlb_add_hstate(shift - PAGE_SHIFT); | ||
660 | } | ||
661 | else | 726 | else |
662 | printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size); | 727 | printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size); |
663 | 728 | ||
@@ -672,16 +737,31 @@ static void zero_ctor(struct kmem_cache *cache, void *addr) | |||
672 | 737 | ||
673 | static int __init hugetlbpage_init(void) | 738 | static int __init hugetlbpage_init(void) |
674 | { | 739 | { |
740 | unsigned int psize; | ||
741 | |||
675 | if (!cpu_has_feature(CPU_FTR_16M_PAGE)) | 742 | if (!cpu_has_feature(CPU_FTR_16M_PAGE)) |
676 | return -ENODEV; | 743 | return -ENODEV; |
677 | 744 | /* Add supported huge page sizes. Need to change HUGE_MAX_HSTATE | |
678 | huge_pgtable_cache = kmem_cache_create("hugepte_cache", | 745 | * and adjust PTE_NONCACHE_NUM if the number of supported huge page |
679 | HUGEPTE_TABLE_SIZE, | 746 | * sizes changes. |
680 | HUGEPTE_TABLE_SIZE, | 747 | */ |
681 | 0, | 748 | set_huge_psize(MMU_PAGE_16M); |
682 | zero_ctor); | 749 | set_huge_psize(MMU_PAGE_64K); |
683 | if (! huge_pgtable_cache) | 750 | set_huge_psize(MMU_PAGE_16G); |
684 | panic("hugetlbpage_init(): could not create hugepte cache\n"); | 751 | |
752 | for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { | ||
753 | if (mmu_huge_psizes[psize]) { | ||
754 | huge_pgtable_cache(psize) = kmem_cache_create( | ||
755 | HUGEPTE_CACHE_NAME(psize), | ||
756 | HUGEPTE_TABLE_SIZE(psize), | ||
757 | HUGEPTE_TABLE_SIZE(psize), | ||
758 | 0, | ||
759 | zero_ctor); | ||
760 | if (!huge_pgtable_cache(psize)) | ||
761 | panic("hugetlbpage_init(): could not create %s"\ | ||
762 | "\n", HUGEPTE_CACHE_NAME(psize)); | ||
763 | } | ||
764 | } | ||
685 | 765 | ||
686 | return 0; | 766 | return 0; |
687 | } | 767 | } |