diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2012-01-06 20:58:22 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2012-01-06 20:58:22 -0500 |
commit | e4e88f31bcb5f05f24b9ae518d4ecb44e1a7774d (patch) | |
tree | 9eef6998f5bbd1a2c999011d9e0151f00c6e7297 /arch/powerpc/mm/hugetlbpage.c | |
parent | 9753dfe19a85e7e45a34a56f4cb2048bb4f50e27 (diff) | |
parent | ef88e3911c0e0301e73fa3b3b2567aabdbe17cc4 (diff) |
Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc
* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc: (185 commits)
powerpc: fix compile error with 85xx/p1010rdb.c
powerpc: fix compile error with 85xx/p1023_rds.c
powerpc/fsl: add MSI support for the Freescale hypervisor
arch/powerpc/sysdev/fsl_rmu.c: introduce missing kfree
powerpc/fsl: Add support for Integrated Flash Controller
powerpc/fsl: update compatiable on fsl 16550 uart nodes
powerpc/85xx: fix PCI and localbus properties in p1022ds.dts
powerpc/85xx: re-enable ePAPR byte channel driver in corenet32_smp_defconfig
powerpc/fsl: Update defconfigs to enable some standard FSL HW features
powerpc: Add TBI PHY node to first MDIO bus
sbc834x: put full compat string in board match check
powerpc/fsl-pci: Allow 64-bit PCIe devices to DMA to any memory address
powerpc: Fix unpaired probe_hcall_entry and probe_hcall_exit
offb: Fix setting of the pseudo-palette for >8bpp
offb: Add palette hack for qemu "standard vga" framebuffer
offb: Fix bug in calculating requested vram size
powerpc/boot: Change the WARN to INFO for boot wrapper overlap message
powerpc/44x: Fix build error on currituck platform
powerpc/boot: Change the load address for the wrapper to fit the kernel
powerpc/44x: Enable CRASH_DUMP for 440x
...
Fix up a trivial conflict in arch/powerpc/include/asm/cputime.h due to
the additional sparse-checking code for cputime_t.
Diffstat (limited to 'arch/powerpc/mm/hugetlbpage.c')
-rw-r--r-- | arch/powerpc/mm/hugetlbpage.c | 116 |
1 files changed, 73 insertions, 43 deletions
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 8558b572e55d..a8b3cc7d90fe 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c | |||
@@ -29,22 +29,22 @@ unsigned int HPAGE_SHIFT; | |||
29 | 29 | ||
30 | /* | 30 | /* |
31 | * Tracks gpages after the device tree is scanned and before the | 31 | * Tracks gpages after the device tree is scanned and before the |
32 | * huge_boot_pages list is ready. On 64-bit implementations, this is | 32 | * huge_boot_pages list is ready. On non-Freescale implementations, this is |
33 | * just used to track 16G pages and so is a single array. 32-bit | 33 | * just used to track 16G pages and so is a single array. FSL-based |
34 | * implementations may have more than one gpage size due to limitations | 34 | * implementations may have more than one gpage size, so we need multiple |
35 | * of the memory allocators, so we need multiple arrays | 35 | * arrays |
36 | */ | 36 | */ |
37 | #ifdef CONFIG_PPC64 | 37 | #ifdef CONFIG_PPC_FSL_BOOK3E |
38 | #define MAX_NUMBER_GPAGES 1024 | ||
39 | static u64 gpage_freearray[MAX_NUMBER_GPAGES]; | ||
40 | static unsigned nr_gpages; | ||
41 | #else | ||
42 | #define MAX_NUMBER_GPAGES 128 | 38 | #define MAX_NUMBER_GPAGES 128 |
43 | struct psize_gpages { | 39 | struct psize_gpages { |
44 | u64 gpage_list[MAX_NUMBER_GPAGES]; | 40 | u64 gpage_list[MAX_NUMBER_GPAGES]; |
45 | unsigned int nr_gpages; | 41 | unsigned int nr_gpages; |
46 | }; | 42 | }; |
47 | static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT]; | 43 | static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT]; |
44 | #else | ||
45 | #define MAX_NUMBER_GPAGES 1024 | ||
46 | static u64 gpage_freearray[MAX_NUMBER_GPAGES]; | ||
47 | static unsigned nr_gpages; | ||
48 | #endif | 48 | #endif |
49 | 49 | ||
50 | static inline int shift_to_mmu_psize(unsigned int shift) | 50 | static inline int shift_to_mmu_psize(unsigned int shift) |
@@ -115,12 +115,12 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, | |||
115 | struct kmem_cache *cachep; | 115 | struct kmem_cache *cachep; |
116 | pte_t *new; | 116 | pte_t *new; |
117 | 117 | ||
118 | #ifdef CONFIG_PPC64 | 118 | #ifdef CONFIG_PPC_FSL_BOOK3E |
119 | cachep = PGT_CACHE(pdshift - pshift); | ||
120 | #else | ||
121 | int i; | 119 | int i; |
122 | int num_hugepd = 1 << (pshift - pdshift); | 120 | int num_hugepd = 1 << (pshift - pdshift); |
123 | cachep = hugepte_cache; | 121 | cachep = hugepte_cache; |
122 | #else | ||
123 | cachep = PGT_CACHE(pdshift - pshift); | ||
124 | #endif | 124 | #endif |
125 | 125 | ||
126 | new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT); | 126 | new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT); |
@@ -132,12 +132,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, | |||
132 | return -ENOMEM; | 132 | return -ENOMEM; |
133 | 133 | ||
134 | spin_lock(&mm->page_table_lock); | 134 | spin_lock(&mm->page_table_lock); |
135 | #ifdef CONFIG_PPC64 | 135 | #ifdef CONFIG_PPC_FSL_BOOK3E |
136 | if (!hugepd_none(*hpdp)) | ||
137 | kmem_cache_free(cachep, new); | ||
138 | else | ||
139 | hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift; | ||
140 | #else | ||
141 | /* | 136 | /* |
142 | * We have multiple higher-level entries that point to the same | 137 | * We have multiple higher-level entries that point to the same |
143 | * actual pte location. Fill in each as we go and backtrack on error. | 138 | * actual pte location. Fill in each as we go and backtrack on error. |
@@ -156,11 +151,28 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, | |||
156 | hpdp->pd = 0; | 151 | hpdp->pd = 0; |
157 | kmem_cache_free(cachep, new); | 152 | kmem_cache_free(cachep, new); |
158 | } | 153 | } |
154 | #else | ||
155 | if (!hugepd_none(*hpdp)) | ||
156 | kmem_cache_free(cachep, new); | ||
157 | else | ||
158 | hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift; | ||
159 | #endif | 159 | #endif |
160 | spin_unlock(&mm->page_table_lock); | 160 | spin_unlock(&mm->page_table_lock); |
161 | return 0; | 161 | return 0; |
162 | } | 162 | } |
163 | 163 | ||
164 | /* | ||
165 | * These macros define how to determine which level of the page table holds | ||
166 | * the hpdp. | ||
167 | */ | ||
168 | #ifdef CONFIG_PPC_FSL_BOOK3E | ||
169 | #define HUGEPD_PGD_SHIFT PGDIR_SHIFT | ||
170 | #define HUGEPD_PUD_SHIFT PUD_SHIFT | ||
171 | #else | ||
172 | #define HUGEPD_PGD_SHIFT PUD_SHIFT | ||
173 | #define HUGEPD_PUD_SHIFT PMD_SHIFT | ||
174 | #endif | ||
175 | |||
164 | pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) | 176 | pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) |
165 | { | 177 | { |
166 | pgd_t *pg; | 178 | pgd_t *pg; |
@@ -173,12 +185,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz | |||
173 | addr &= ~(sz-1); | 185 | addr &= ~(sz-1); |
174 | 186 | ||
175 | pg = pgd_offset(mm, addr); | 187 | pg = pgd_offset(mm, addr); |
176 | if (pshift >= PUD_SHIFT) { | 188 | |
189 | if (pshift >= HUGEPD_PGD_SHIFT) { | ||
177 | hpdp = (hugepd_t *)pg; | 190 | hpdp = (hugepd_t *)pg; |
178 | } else { | 191 | } else { |
179 | pdshift = PUD_SHIFT; | 192 | pdshift = PUD_SHIFT; |
180 | pu = pud_alloc(mm, pg, addr); | 193 | pu = pud_alloc(mm, pg, addr); |
181 | if (pshift >= PMD_SHIFT) { | 194 | if (pshift >= HUGEPD_PUD_SHIFT) { |
182 | hpdp = (hugepd_t *)pu; | 195 | hpdp = (hugepd_t *)pu; |
183 | } else { | 196 | } else { |
184 | pdshift = PMD_SHIFT; | 197 | pdshift = PMD_SHIFT; |
@@ -198,7 +211,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz | |||
198 | return hugepte_offset(hpdp, addr, pdshift); | 211 | return hugepte_offset(hpdp, addr, pdshift); |
199 | } | 212 | } |
200 | 213 | ||
201 | #ifdef CONFIG_PPC32 | 214 | #ifdef CONFIG_PPC_FSL_BOOK3E |
202 | /* Build list of addresses of gigantic pages. This function is used in early | 215 | /* Build list of addresses of gigantic pages. This function is used in early |
203 | * boot before the buddy or bootmem allocator is setup. | 216 | * boot before the buddy or bootmem allocator is setup. |
204 | */ | 217 | */ |
@@ -318,7 +331,7 @@ void __init reserve_hugetlb_gpages(void) | |||
318 | } | 331 | } |
319 | } | 332 | } |
320 | 333 | ||
321 | #else /* PPC64 */ | 334 | #else /* !PPC_FSL_BOOK3E */ |
322 | 335 | ||
323 | /* Build list of addresses of gigantic pages. This function is used in early | 336 | /* Build list of addresses of gigantic pages. This function is used in early |
324 | * boot before the buddy or bootmem allocator is setup. | 337 | * boot before the buddy or bootmem allocator is setup. |
@@ -356,7 +369,7 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) | |||
356 | return 0; | 369 | return 0; |
357 | } | 370 | } |
358 | 371 | ||
359 | #ifdef CONFIG_PPC32 | 372 | #ifdef CONFIG_PPC_FSL_BOOK3E |
360 | #define HUGEPD_FREELIST_SIZE \ | 373 | #define HUGEPD_FREELIST_SIZE \ |
361 | ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t)) | 374 | ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t)) |
362 | 375 | ||
@@ -416,11 +429,11 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif | |||
416 | unsigned long pdmask = ~((1UL << pdshift) - 1); | 429 | unsigned long pdmask = ~((1UL << pdshift) - 1); |
417 | unsigned int num_hugepd = 1; | 430 | unsigned int num_hugepd = 1; |
418 | 431 | ||
419 | #ifdef CONFIG_PPC64 | 432 | #ifdef CONFIG_PPC_FSL_BOOK3E |
420 | unsigned int shift = hugepd_shift(*hpdp); | 433 | /* Note: On fsl the hpdp may be the first of several */ |
421 | #else | ||
422 | /* Note: On 32-bit the hpdp may be the first of several */ | ||
423 | num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift)); | 434 | num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift)); |
435 | #else | ||
436 | unsigned int shift = hugepd_shift(*hpdp); | ||
424 | #endif | 437 | #endif |
425 | 438 | ||
426 | start &= pdmask; | 439 | start &= pdmask; |
@@ -438,10 +451,11 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif | |||
438 | hpdp->pd = 0; | 451 | hpdp->pd = 0; |
439 | 452 | ||
440 | tlb->need_flush = 1; | 453 | tlb->need_flush = 1; |
441 | #ifdef CONFIG_PPC64 | 454 | |
442 | pgtable_free_tlb(tlb, hugepte, pdshift - shift); | 455 | #ifdef CONFIG_PPC_FSL_BOOK3E |
443 | #else | ||
444 | hugepd_free(tlb, hugepte); | 456 | hugepd_free(tlb, hugepte); |
457 | #else | ||
458 | pgtable_free_tlb(tlb, hugepte, pdshift - shift); | ||
445 | #endif | 459 | #endif |
446 | } | 460 | } |
447 | 461 | ||
@@ -454,14 +468,23 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, | |||
454 | unsigned long start; | 468 | unsigned long start; |
455 | 469 | ||
456 | start = addr; | 470 | start = addr; |
457 | pmd = pmd_offset(pud, addr); | ||
458 | do { | 471 | do { |
472 | pmd = pmd_offset(pud, addr); | ||
459 | next = pmd_addr_end(addr, end); | 473 | next = pmd_addr_end(addr, end); |
460 | if (pmd_none(*pmd)) | 474 | if (pmd_none(*pmd)) |
461 | continue; | 475 | continue; |
476 | #ifdef CONFIG_PPC_FSL_BOOK3E | ||
477 | /* | ||
478 | * Increment next by the size of the huge mapping since | ||
479 | * there may be more than one entry at this level for a | ||
480 | * single hugepage, but all of them point to | ||
481 | * the same kmem cache that holds the hugepte. | ||
482 | */ | ||
483 | next = addr + (1 << hugepd_shift(*(hugepd_t *)pmd)); | ||
484 | #endif | ||
462 | free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT, | 485 | free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT, |
463 | addr, next, floor, ceiling); | 486 | addr, next, floor, ceiling); |
464 | } while (pmd++, addr = next, addr != end); | 487 | } while (addr = next, addr != end); |
465 | 488 | ||
466 | start &= PUD_MASK; | 489 | start &= PUD_MASK; |
467 | if (start < floor) | 490 | if (start < floor) |
@@ -488,8 +511,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, | |||
488 | unsigned long start; | 511 | unsigned long start; |
489 | 512 | ||
490 | start = addr; | 513 | start = addr; |
491 | pud = pud_offset(pgd, addr); | ||
492 | do { | 514 | do { |
515 | pud = pud_offset(pgd, addr); | ||
493 | next = pud_addr_end(addr, end); | 516 | next = pud_addr_end(addr, end); |
494 | if (!is_hugepd(pud)) { | 517 | if (!is_hugepd(pud)) { |
495 | if (pud_none_or_clear_bad(pud)) | 518 | if (pud_none_or_clear_bad(pud)) |
@@ -497,10 +520,19 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, | |||
497 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, | 520 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, |
498 | ceiling); | 521 | ceiling); |
499 | } else { | 522 | } else { |
523 | #ifdef CONFIG_PPC_FSL_BOOK3E | ||
524 | /* | ||
525 | * Increment next by the size of the huge mapping since | ||
526 | * there may be more than one entry at this level for a | ||
527 | * single hugepage, but all of them point to | ||
528 | * the same kmem cache that holds the hugepte. | ||
529 | */ | ||
530 | next = addr + (1 << hugepd_shift(*(hugepd_t *)pud)); | ||
531 | #endif | ||
500 | free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT, | 532 | free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT, |
501 | addr, next, floor, ceiling); | 533 | addr, next, floor, ceiling); |
502 | } | 534 | } |
503 | } while (pud++, addr = next, addr != end); | 535 | } while (addr = next, addr != end); |
504 | 536 | ||
505 | start &= PGDIR_MASK; | 537 | start &= PGDIR_MASK; |
506 | if (start < floor) | 538 | if (start < floor) |
@@ -555,12 +587,12 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, | |||
555 | continue; | 587 | continue; |
556 | hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); | 588 | hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); |
557 | } else { | 589 | } else { |
558 | #ifdef CONFIG_PPC32 | 590 | #ifdef CONFIG_PPC_FSL_BOOK3E |
559 | /* | 591 | /* |
560 | * Increment next by the size of the huge mapping since | 592 | * Increment next by the size of the huge mapping since |
561 | * on 32-bit there may be more than one entry at the pgd | 593 | * there may be more than one entry at the pgd level |
562 | * level for a single hugepage, but all of them point to | 594 | * for a single hugepage, but all of them point to the |
563 | * the same kmem cache that holds the hugepte. | 595 | * same kmem cache that holds the hugepte. |
564 | */ | 596 | */ |
565 | next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd)); | 597 | next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd)); |
566 | #endif | 598 | #endif |
@@ -698,19 +730,17 @@ int gup_hugepd(hugepd_t *hugepd, unsigned pdshift, | |||
698 | return 1; | 730 | return 1; |
699 | } | 731 | } |
700 | 732 | ||
733 | #ifdef CONFIG_PPC_MM_SLICES | ||
701 | unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, | 734 | unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, |
702 | unsigned long len, unsigned long pgoff, | 735 | unsigned long len, unsigned long pgoff, |
703 | unsigned long flags) | 736 | unsigned long flags) |
704 | { | 737 | { |
705 | #ifdef CONFIG_PPC_MM_SLICES | ||
706 | struct hstate *hstate = hstate_file(file); | 738 | struct hstate *hstate = hstate_file(file); |
707 | int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); | 739 | int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); |
708 | 740 | ||
709 | return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0); | 741 | return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0); |
710 | #else | ||
711 | return get_unmapped_area(file, addr, len, pgoff, flags); | ||
712 | #endif | ||
713 | } | 742 | } |
743 | #endif | ||
714 | 744 | ||
715 | unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) | 745 | unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) |
716 | { | 746 | { |
@@ -784,7 +814,7 @@ static int __init hugepage_setup_sz(char *str) | |||
784 | } | 814 | } |
785 | __setup("hugepagesz=", hugepage_setup_sz); | 815 | __setup("hugepagesz=", hugepage_setup_sz); |
786 | 816 | ||
787 | #ifdef CONFIG_FSL_BOOKE | 817 | #ifdef CONFIG_PPC_FSL_BOOK3E |
788 | struct kmem_cache *hugepte_cache; | 818 | struct kmem_cache *hugepte_cache; |
789 | static int __init hugetlbpage_init(void) | 819 | static int __init hugetlbpage_init(void) |
790 | { | 820 | { |