aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm/hugetlbpage.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2012-01-06 20:58:22 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2012-01-06 20:58:22 -0500
commite4e88f31bcb5f05f24b9ae518d4ecb44e1a7774d (patch)
tree9eef6998f5bbd1a2c999011d9e0151f00c6e7297 /arch/powerpc/mm/hugetlbpage.c
parent9753dfe19a85e7e45a34a56f4cb2048bb4f50e27 (diff)
parentef88e3911c0e0301e73fa3b3b2567aabdbe17cc4 (diff)
Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc
* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc: (185 commits) powerpc: fix compile error with 85xx/p1010rdb.c powerpc: fix compile error with 85xx/p1023_rds.c powerpc/fsl: add MSI support for the Freescale hypervisor arch/powerpc/sysdev/fsl_rmu.c: introduce missing kfree powerpc/fsl: Add support for Integrated Flash Controller powerpc/fsl: update compatiable on fsl 16550 uart nodes powerpc/85xx: fix PCI and localbus properties in p1022ds.dts powerpc/85xx: re-enable ePAPR byte channel driver in corenet32_smp_defconfig powerpc/fsl: Update defconfigs to enable some standard FSL HW features powerpc: Add TBI PHY node to first MDIO bus sbc834x: put full compat string in board match check powerpc/fsl-pci: Allow 64-bit PCIe devices to DMA to any memory address powerpc: Fix unpaired probe_hcall_entry and probe_hcall_exit offb: Fix setting of the pseudo-palette for >8bpp offb: Add palette hack for qemu "standard vga" framebuffer offb: Fix bug in calculating requested vram size powerpc/boot: Change the WARN to INFO for boot wrapper overlap message powerpc/44x: Fix build error on currituck platform powerpc/boot: Change the load address for the wrapper to fit the kernel powerpc/44x: Enable CRASH_DUMP for 440x ... Fix up a trivial conflict in arch/powerpc/include/asm/cputime.h due to the additional sparse-checking code for cputime_t.
Diffstat (limited to 'arch/powerpc/mm/hugetlbpage.c')
-rw-r--r--arch/powerpc/mm/hugetlbpage.c116
1 files changed, 73 insertions, 43 deletions
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 8558b572e55d..a8b3cc7d90fe 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -29,22 +29,22 @@ unsigned int HPAGE_SHIFT;
29 29
30/* 30/*
31 * Tracks gpages after the device tree is scanned and before the 31 * Tracks gpages after the device tree is scanned and before the
32 * huge_boot_pages list is ready. On 64-bit implementations, this is 32 * huge_boot_pages list is ready. On non-Freescale implementations, this is
33 * just used to track 16G pages and so is a single array. 32-bit 33 * just used to track 16G pages and so is a single array. FSL-based
34 * implementations may have more than one gpage size due to limitations 34 * implementations may have more than one gpage size, so we need multiple
35 * of the memory allocators, so we need multiple arrays 35 * arrays
36 */ 36 */
37#ifdef CONFIG_PPC64 37#ifdef CONFIG_PPC_FSL_BOOK3E
38#define MAX_NUMBER_GPAGES 1024
39static u64 gpage_freearray[MAX_NUMBER_GPAGES];
40static unsigned nr_gpages;
41#else
42#define MAX_NUMBER_GPAGES 128 38#define MAX_NUMBER_GPAGES 128
43struct psize_gpages { 39struct psize_gpages {
44 u64 gpage_list[MAX_NUMBER_GPAGES]; 40 u64 gpage_list[MAX_NUMBER_GPAGES];
45 unsigned int nr_gpages; 41 unsigned int nr_gpages;
46}; 42};
47static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT]; 43static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT];
44#else
45#define MAX_NUMBER_GPAGES 1024
46static u64 gpage_freearray[MAX_NUMBER_GPAGES];
47static unsigned nr_gpages;
48#endif 48#endif
49 49
50static inline int shift_to_mmu_psize(unsigned int shift) 50static inline int shift_to_mmu_psize(unsigned int shift)
@@ -115,12 +115,12 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
115 struct kmem_cache *cachep; 115 struct kmem_cache *cachep;
116 pte_t *new; 116 pte_t *new;
117 117
118#ifdef CONFIG_PPC64 118#ifdef CONFIG_PPC_FSL_BOOK3E
119 cachep = PGT_CACHE(pdshift - pshift);
120#else
121 int i; 119 int i;
122 int num_hugepd = 1 << (pshift - pdshift); 120 int num_hugepd = 1 << (pshift - pdshift);
123 cachep = hugepte_cache; 121 cachep = hugepte_cache;
122#else
123 cachep = PGT_CACHE(pdshift - pshift);
124#endif 124#endif
125 125
126 new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT); 126 new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT);
@@ -132,12 +132,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
132 return -ENOMEM; 132 return -ENOMEM;
133 133
134 spin_lock(&mm->page_table_lock); 134 spin_lock(&mm->page_table_lock);
135#ifdef CONFIG_PPC64 135#ifdef CONFIG_PPC_FSL_BOOK3E
136 if (!hugepd_none(*hpdp))
137 kmem_cache_free(cachep, new);
138 else
139 hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
140#else
141 /* 136 /*
142 * We have multiple higher-level entries that point to the same 137 * We have multiple higher-level entries that point to the same
143 * actual pte location. Fill in each as we go and backtrack on error. 138 * actual pte location. Fill in each as we go and backtrack on error.
@@ -156,11 +151,28 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
156 hpdp->pd = 0; 151 hpdp->pd = 0;
157 kmem_cache_free(cachep, new); 152 kmem_cache_free(cachep, new);
158 } 153 }
154#else
155 if (!hugepd_none(*hpdp))
156 kmem_cache_free(cachep, new);
157 else
158 hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
159#endif 159#endif
160 spin_unlock(&mm->page_table_lock); 160 spin_unlock(&mm->page_table_lock);
161 return 0; 161 return 0;
162} 162}
163 163
164/*
165 * These macros define how to determine which level of the page table holds
166 * the hpdp.
167 */
168#ifdef CONFIG_PPC_FSL_BOOK3E
169#define HUGEPD_PGD_SHIFT PGDIR_SHIFT
170#define HUGEPD_PUD_SHIFT PUD_SHIFT
171#else
172#define HUGEPD_PGD_SHIFT PUD_SHIFT
173#define HUGEPD_PUD_SHIFT PMD_SHIFT
174#endif
175
164pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) 176pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
165{ 177{
166 pgd_t *pg; 178 pgd_t *pg;
@@ -173,12 +185,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
173 addr &= ~(sz-1); 185 addr &= ~(sz-1);
174 186
175 pg = pgd_offset(mm, addr); 187 pg = pgd_offset(mm, addr);
176 if (pshift >= PUD_SHIFT) { 188
189 if (pshift >= HUGEPD_PGD_SHIFT) {
177 hpdp = (hugepd_t *)pg; 190 hpdp = (hugepd_t *)pg;
178 } else { 191 } else {
179 pdshift = PUD_SHIFT; 192 pdshift = PUD_SHIFT;
180 pu = pud_alloc(mm, pg, addr); 193 pu = pud_alloc(mm, pg, addr);
181 if (pshift >= PMD_SHIFT) { 194 if (pshift >= HUGEPD_PUD_SHIFT) {
182 hpdp = (hugepd_t *)pu; 195 hpdp = (hugepd_t *)pu;
183 } else { 196 } else {
184 pdshift = PMD_SHIFT; 197 pdshift = PMD_SHIFT;
@@ -198,7 +211,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
198 return hugepte_offset(hpdp, addr, pdshift); 211 return hugepte_offset(hpdp, addr, pdshift);
199} 212}
200 213
201#ifdef CONFIG_PPC32 214#ifdef CONFIG_PPC_FSL_BOOK3E
202/* Build list of addresses of gigantic pages. This function is used in early 215/* Build list of addresses of gigantic pages. This function is used in early
203 * boot before the buddy or bootmem allocator is setup. 216 * boot before the buddy or bootmem allocator is setup.
204 */ 217 */
@@ -318,7 +331,7 @@ void __init reserve_hugetlb_gpages(void)
318 } 331 }
319} 332}
320 333
321#else /* PPC64 */ 334#else /* !PPC_FSL_BOOK3E */
322 335
323/* Build list of addresses of gigantic pages. This function is used in early 336/* Build list of addresses of gigantic pages. This function is used in early
324 * boot before the buddy or bootmem allocator is setup. 337 * boot before the buddy or bootmem allocator is setup.
@@ -356,7 +369,7 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
356 return 0; 369 return 0;
357} 370}
358 371
359#ifdef CONFIG_PPC32 372#ifdef CONFIG_PPC_FSL_BOOK3E
360#define HUGEPD_FREELIST_SIZE \ 373#define HUGEPD_FREELIST_SIZE \
361 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t)) 374 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
362 375
@@ -416,11 +429,11 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif
416 unsigned long pdmask = ~((1UL << pdshift) - 1); 429 unsigned long pdmask = ~((1UL << pdshift) - 1);
417 unsigned int num_hugepd = 1; 430 unsigned int num_hugepd = 1;
418 431
419#ifdef CONFIG_PPC64 432#ifdef CONFIG_PPC_FSL_BOOK3E
420 unsigned int shift = hugepd_shift(*hpdp); 433 /* Note: On fsl the hpdp may be the first of several */
421#else
422 /* Note: On 32-bit the hpdp may be the first of several */
423 num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift)); 434 num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift));
435#else
436 unsigned int shift = hugepd_shift(*hpdp);
424#endif 437#endif
425 438
426 start &= pdmask; 439 start &= pdmask;
@@ -438,10 +451,11 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif
438 hpdp->pd = 0; 451 hpdp->pd = 0;
439 452
440 tlb->need_flush = 1; 453 tlb->need_flush = 1;
441#ifdef CONFIG_PPC64 454
442 pgtable_free_tlb(tlb, hugepte, pdshift - shift); 455#ifdef CONFIG_PPC_FSL_BOOK3E
443#else
444 hugepd_free(tlb, hugepte); 456 hugepd_free(tlb, hugepte);
457#else
458 pgtable_free_tlb(tlb, hugepte, pdshift - shift);
445#endif 459#endif
446} 460}
447 461
@@ -454,14 +468,23 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
454 unsigned long start; 468 unsigned long start;
455 469
456 start = addr; 470 start = addr;
457 pmd = pmd_offset(pud, addr);
458 do { 471 do {
472 pmd = pmd_offset(pud, addr);
459 next = pmd_addr_end(addr, end); 473 next = pmd_addr_end(addr, end);
460 if (pmd_none(*pmd)) 474 if (pmd_none(*pmd))
461 continue; 475 continue;
476#ifdef CONFIG_PPC_FSL_BOOK3E
477 /*
478 * Increment next by the size of the huge mapping since
479 * there may be more than one entry at this level for a
480 * single hugepage, but all of them point to
481 * the same kmem cache that holds the hugepte.
482 */
483 next = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
484#endif
462 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT, 485 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
463 addr, next, floor, ceiling); 486 addr, next, floor, ceiling);
464 } while (pmd++, addr = next, addr != end); 487 } while (addr = next, addr != end);
465 488
466 start &= PUD_MASK; 489 start &= PUD_MASK;
467 if (start < floor) 490 if (start < floor)
@@ -488,8 +511,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
488 unsigned long start; 511 unsigned long start;
489 512
490 start = addr; 513 start = addr;
491 pud = pud_offset(pgd, addr);
492 do { 514 do {
515 pud = pud_offset(pgd, addr);
493 next = pud_addr_end(addr, end); 516 next = pud_addr_end(addr, end);
494 if (!is_hugepd(pud)) { 517 if (!is_hugepd(pud)) {
495 if (pud_none_or_clear_bad(pud)) 518 if (pud_none_or_clear_bad(pud))
@@ -497,10 +520,19 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
497 hugetlb_free_pmd_range(tlb, pud, addr, next, floor, 520 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
498 ceiling); 521 ceiling);
499 } else { 522 } else {
523#ifdef CONFIG_PPC_FSL_BOOK3E
524 /*
525 * Increment next by the size of the huge mapping since
526 * there may be more than one entry at this level for a
527 * single hugepage, but all of them point to
528 * the same kmem cache that holds the hugepte.
529 */
530 next = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
531#endif
500 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT, 532 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
501 addr, next, floor, ceiling); 533 addr, next, floor, ceiling);
502 } 534 }
503 } while (pud++, addr = next, addr != end); 535 } while (addr = next, addr != end);
504 536
505 start &= PGDIR_MASK; 537 start &= PGDIR_MASK;
506 if (start < floor) 538 if (start < floor)
@@ -555,12 +587,12 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
555 continue; 587 continue;
556 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); 588 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
557 } else { 589 } else {
558#ifdef CONFIG_PPC32 590#ifdef CONFIG_PPC_FSL_BOOK3E
559 /* 591 /*
560 * Increment next by the size of the huge mapping since 592 * Increment next by the size of the huge mapping since
561 * on 32-bit there may be more than one entry at the pgd 593 * there may be more than one entry at the pgd level
562 * level for a single hugepage, but all of them point to 594 * for a single hugepage, but all of them point to the
563 * the same kmem cache that holds the hugepte. 595 * same kmem cache that holds the hugepte.
564 */ 596 */
565 next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd)); 597 next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
566#endif 598#endif
@@ -698,19 +730,17 @@ int gup_hugepd(hugepd_t *hugepd, unsigned pdshift,
698 return 1; 730 return 1;
699} 731}
700 732
733#ifdef CONFIG_PPC_MM_SLICES
701unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, 734unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
702 unsigned long len, unsigned long pgoff, 735 unsigned long len, unsigned long pgoff,
703 unsigned long flags) 736 unsigned long flags)
704{ 737{
705#ifdef CONFIG_PPC_MM_SLICES
706 struct hstate *hstate = hstate_file(file); 738 struct hstate *hstate = hstate_file(file);
707 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); 739 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
708 740
709 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0); 741 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0);
710#else
711 return get_unmapped_area(file, addr, len, pgoff, flags);
712#endif
713} 742}
743#endif
714 744
715unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 745unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
716{ 746{
@@ -784,7 +814,7 @@ static int __init hugepage_setup_sz(char *str)
784} 814}
785__setup("hugepagesz=", hugepage_setup_sz); 815__setup("hugepagesz=", hugepage_setup_sz);
786 816
787#ifdef CONFIG_FSL_BOOKE 817#ifdef CONFIG_PPC_FSL_BOOK3E
788struct kmem_cache *hugepte_cache; 818struct kmem_cache *hugepte_cache;
789static int __init hugetlbpage_init(void) 819static int __init hugetlbpage_init(void)
790{ 820{