aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/mm')
-rw-r--r--arch/powerpc/mm/44x_mmu.c29
-rw-r--r--arch/powerpc/mm/fault.c3
-rw-r--r--arch/powerpc/mm/hash_low_64.S17
-rw-r--r--arch/powerpc/mm/hash_utils_64.c42
-rw-r--r--arch/powerpc/mm/hugetlbpage.c13
-rw-r--r--arch/powerpc/mm/init_32.c7
-rw-r--r--arch/powerpc/mm/init_64.c4
-rw-r--r--arch/powerpc/mm/mem.c9
-rw-r--r--arch/powerpc/mm/mmu_decl.h6
-rw-r--r--arch/powerpc/mm/numa.c310
-rw-r--r--arch/powerpc/mm/pgtable_32.c6
-rw-r--r--arch/powerpc/mm/ppc_mmu_32.c27
-rw-r--r--arch/powerpc/mm/slice.c177
-rw-r--r--arch/powerpc/mm/stab.c4
-rw-r--r--arch/powerpc/mm/tlb_64.c7
15 files changed, 494 insertions, 167 deletions
diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c
index 953fb919eb06..98052ac96580 100644
--- a/arch/powerpc/mm/44x_mmu.c
+++ b/arch/powerpc/mm/44x_mmu.c
@@ -27,6 +27,7 @@
27#include <asm/mmu.h> 27#include <asm/mmu.h>
28#include <asm/system.h> 28#include <asm/system.h>
29#include <asm/page.h> 29#include <asm/page.h>
30#include <asm/cacheflush.h>
30 31
31#include "mmu_decl.h" 32#include "mmu_decl.h"
32 33
@@ -37,11 +38,35 @@ unsigned int tlb_44x_index; /* = 0 */
37unsigned int tlb_44x_hwater = PPC44x_TLB_SIZE - 1 - PPC44x_EARLY_TLBS; 38unsigned int tlb_44x_hwater = PPC44x_TLB_SIZE - 1 - PPC44x_EARLY_TLBS;
38int icache_44x_need_flush; 39int icache_44x_need_flush;
39 40
41static void __init ppc44x_update_tlb_hwater(void)
42{
43 extern unsigned int tlb_44x_patch_hwater_D[];
44 extern unsigned int tlb_44x_patch_hwater_I[];
45
46 /* The TLB miss handlers hard codes the watermark in a cmpli
47 * instruction to improve performances rather than loading it
48 * from the global variable. Thus, we patch the instructions
49 * in the 2 TLB miss handlers when updating the value
50 */
51 tlb_44x_patch_hwater_D[0] = (tlb_44x_patch_hwater_D[0] & 0xffff0000) |
52 tlb_44x_hwater;
53 flush_icache_range((unsigned long)&tlb_44x_patch_hwater_D[0],
54 (unsigned long)&tlb_44x_patch_hwater_D[1]);
55 tlb_44x_patch_hwater_I[0] = (tlb_44x_patch_hwater_I[0] & 0xffff0000) |
56 tlb_44x_hwater;
57 flush_icache_range((unsigned long)&tlb_44x_patch_hwater_I[0],
58 (unsigned long)&tlb_44x_patch_hwater_I[1]);
59}
60
40/* 61/*
41 * "Pins" a 256MB TLB entry in AS0 for kernel lowmem 62 * "Pins" a 256MB TLB entry in AS0 for kernel lowmem
42 */ 63 */
43static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys) 64static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys)
44{ 65{
66 unsigned int entry = tlb_44x_hwater--;
67
68 ppc44x_update_tlb_hwater();
69
45 __asm__ __volatile__( 70 __asm__ __volatile__(
46 "tlbwe %2,%3,%4\n" 71 "tlbwe %2,%3,%4\n"
47 "tlbwe %1,%3,%5\n" 72 "tlbwe %1,%3,%5\n"
@@ -50,7 +75,7 @@ static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys)
50 : "r" (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G), 75 : "r" (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G),
51 "r" (phys), 76 "r" (phys),
52 "r" (virt | PPC44x_TLB_VALID | PPC44x_TLB_256M), 77 "r" (virt | PPC44x_TLB_VALID | PPC44x_TLB_256M),
53 "r" (tlb_44x_hwater--), /* slot for this TLB entry */ 78 "r" (entry),
54 "i" (PPC44x_TLB_PAGEID), 79 "i" (PPC44x_TLB_PAGEID),
55 "i" (PPC44x_TLB_XLAT), 80 "i" (PPC44x_TLB_XLAT),
56 "i" (PPC44x_TLB_ATTRIB)); 81 "i" (PPC44x_TLB_ATTRIB));
@@ -58,6 +83,8 @@ static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys)
58 83
59void __init MMU_init_hw(void) 84void __init MMU_init_hw(void)
60{ 85{
86 ppc44x_update_tlb_hwater();
87
61 flush_instruction_cache(); 88 flush_instruction_cache();
62} 89}
63 90
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 7b2510799266..1707d00331fc 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -306,7 +306,8 @@ good_area:
306 flush_dcache_icache_page(page); 306 flush_dcache_icache_page(page);
307 set_bit(PG_arch_1, &page->flags); 307 set_bit(PG_arch_1, &page->flags);
308 } 308 }
309 pte_update(ptep, 0, _PAGE_HWEXEC); 309 pte_update(ptep, 0, _PAGE_HWEXEC |
310 _PAGE_ACCESSED);
310 _tlbie(address, mm->context.id); 311 _tlbie(address, mm->context.id);
311 pte_unmap_unlock(ptep, ptl); 312 pte_unmap_unlock(ptep, ptl);
312 up_read(&mm->mmap_sem); 313 up_read(&mm->mmap_sem);
diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S
index 70f4c833fa32..a719f53921a5 100644
--- a/arch/powerpc/mm/hash_low_64.S
+++ b/arch/powerpc/mm/hash_low_64.S
@@ -388,7 +388,7 @@ _GLOBAL(__hash_page_4K)
388 */ 388 */
389 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 389 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
390 or r30,r30,r31 390 or r30,r30,r31
391 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 391 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
392 oris r30,r30,_PAGE_COMBO@h 392 oris r30,r30,_PAGE_COMBO@h
393 /* Write the linux PTE atomically (setting busy) */ 393 /* Write the linux PTE atomically (setting busy) */
394 stdcx. r30,0,r6 394 stdcx. r30,0,r6
@@ -468,7 +468,7 @@ END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
468 * go to out-of-line code to try to modify the HPTE. We look for 468 * go to out-of-line code to try to modify the HPTE. We look for
469 * the bit at (1 >> (index + 32)) 469 * the bit at (1 >> (index + 32))
470 */ 470 */
471 andi. r0,r31,_PAGE_HASHPTE 471 rldicl. r0,r31,64-12,48
472 li r26,0 /* Default hidx */ 472 li r26,0 /* Default hidx */
473 beq htab_insert_pte 473 beq htab_insert_pte
474 474
@@ -726,11 +726,11 @@ BEGIN_FTR_SECTION
726 bne- ht64_bail_ok 726 bne- ht64_bail_ok
727END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) 727END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
728 /* Prepare new PTE value (turn access RW into DIRTY, then 728 /* Prepare new PTE value (turn access RW into DIRTY, then
729 * add BUSY,HASHPTE and ACCESSED) 729 * add BUSY and ACCESSED)
730 */ 730 */
731 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 731 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
732 or r30,r30,r31 732 or r30,r30,r31
733 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 733 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
734 /* Write the linux PTE atomically (setting busy) */ 734 /* Write the linux PTE atomically (setting busy) */
735 stdcx. r30,0,r6 735 stdcx. r30,0,r6
736 bne- 1b 736 bne- 1b
@@ -798,18 +798,21 @@ END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
798 /* Check if we may already be in the hashtable, in this case, we 798 /* Check if we may already be in the hashtable, in this case, we
799 * go to out-of-line code to try to modify the HPTE 799 * go to out-of-line code to try to modify the HPTE
800 */ 800 */
801 andi. r0,r31,_PAGE_HASHPTE 801 rldicl. r0,r31,64-12,48
802 bne ht64_modify_pte 802 bne ht64_modify_pte
803 803
804ht64_insert_pte: 804ht64_insert_pte:
805 /* Clear hpte bits in new pte (we also clear BUSY btw) and 805 /* Clear hpte bits in new pte (we also clear BUSY btw) and
806 * add _PAGE_HASHPTE 806 * add _PAGE_HPTE_SUB0
807 */ 807 */
808 lis r0,_PAGE_HPTEFLAGS@h 808 lis r0,_PAGE_HPTEFLAGS@h
809 ori r0,r0,_PAGE_HPTEFLAGS@l 809 ori r0,r0,_PAGE_HPTEFLAGS@l
810 andc r30,r30,r0 810 andc r30,r30,r0
811#ifdef CONFIG_PPC_64K_PAGES
812 oris r30,r30,_PAGE_HPTE_SUB0@h
813#else
811 ori r30,r30,_PAGE_HASHPTE 814 ori r30,r30,_PAGE_HASHPTE
812 815#endif
813 /* Phyical address in r5 */ 816 /* Phyical address in r5 */
814 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 817 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
815 sldi r5,r5,PAGE_SHIFT 818 sldi r5,r5,PAGE_SHIFT
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 0f2d239d94c4..8d3b58ebd38e 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -120,7 +120,7 @@ static DEFINE_SPINLOCK(linear_map_hash_lock);
120 120
121/* Pre-POWER4 CPUs (4k pages only) 121/* Pre-POWER4 CPUs (4k pages only)
122 */ 122 */
123struct mmu_psize_def mmu_psize_defaults_old[] = { 123static struct mmu_psize_def mmu_psize_defaults_old[] = {
124 [MMU_PAGE_4K] = { 124 [MMU_PAGE_4K] = {
125 .shift = 12, 125 .shift = 12,
126 .sllp = 0, 126 .sllp = 0,
@@ -134,7 +134,7 @@ struct mmu_psize_def mmu_psize_defaults_old[] = {
134 * 134 *
135 * Support for 16Mb large pages 135 * Support for 16Mb large pages
136 */ 136 */
137struct mmu_psize_def mmu_psize_defaults_gp[] = { 137static struct mmu_psize_def mmu_psize_defaults_gp[] = {
138 [MMU_PAGE_4K] = { 138 [MMU_PAGE_4K] = {
139 .shift = 12, 139 .shift = 12,
140 .sllp = 0, 140 .sllp = 0,
@@ -533,8 +533,6 @@ void __init htab_initialize(void)
533 unsigned long base = 0, size = 0, limit; 533 unsigned long base = 0, size = 0, limit;
534 int i; 534 int i;
535 535
536 extern unsigned long tce_alloc_start, tce_alloc_end;
537
538 DBG(" -> htab_initialize()\n"); 536 DBG(" -> htab_initialize()\n");
539 537
540 /* Initialize segment sizes */ 538 /* Initialize segment sizes */
@@ -697,6 +695,28 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
697 return pp; 695 return pp;
698} 696}
699 697
698#ifdef CONFIG_PPC_MM_SLICES
699unsigned int get_paca_psize(unsigned long addr)
700{
701 unsigned long index, slices;
702
703 if (addr < SLICE_LOW_TOP) {
704 slices = get_paca()->context.low_slices_psize;
705 index = GET_LOW_SLICE_INDEX(addr);
706 } else {
707 slices = get_paca()->context.high_slices_psize;
708 index = GET_HIGH_SLICE_INDEX(addr);
709 }
710 return (slices >> (index * 4)) & 0xF;
711}
712
713#else
714unsigned int get_paca_psize(unsigned long addr)
715{
716 return get_paca()->context.user_psize;
717}
718#endif
719
700/* 720/*
701 * Demote a segment to using 4k pages. 721 * Demote a segment to using 4k pages.
702 * For now this makes the whole process use 4k pages. 722 * For now this makes the whole process use 4k pages.
@@ -704,13 +724,13 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
704#ifdef CONFIG_PPC_64K_PAGES 724#ifdef CONFIG_PPC_64K_PAGES
705void demote_segment_4k(struct mm_struct *mm, unsigned long addr) 725void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
706{ 726{
707 if (mm->context.user_psize == MMU_PAGE_4K) 727 if (get_slice_psize(mm, addr) == MMU_PAGE_4K)
708 return; 728 return;
709 slice_set_user_psize(mm, MMU_PAGE_4K); 729 slice_set_range_psize(mm, addr, 1, MMU_PAGE_4K);
710#ifdef CONFIG_SPU_BASE 730#ifdef CONFIG_SPU_BASE
711 spu_flush_all_slbs(mm); 731 spu_flush_all_slbs(mm);
712#endif 732#endif
713 if (get_paca()->context.user_psize != MMU_PAGE_4K) { 733 if (get_paca_psize(addr) != MMU_PAGE_4K) {
714 get_paca()->context = mm->context; 734 get_paca()->context = mm->context;
715 slb_flush_and_rebolt(); 735 slb_flush_and_rebolt();
716 } 736 }
@@ -794,11 +814,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
794 DBG_LOW(" user region with no mm !\n"); 814 DBG_LOW(" user region with no mm !\n");
795 return 1; 815 return 1;
796 } 816 }
797#ifdef CONFIG_PPC_MM_SLICES
798 psize = get_slice_psize(mm, ea); 817 psize = get_slice_psize(mm, ea);
799#else
800 psize = mm->context.user_psize;
801#endif
802 ssize = user_segment_size(ea); 818 ssize = user_segment_size(ea);
803 vsid = get_vsid(mm->context.id, ea, ssize); 819 vsid = get_vsid(mm->context.id, ea, ssize);
804 break; 820 break;
@@ -870,7 +886,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
870 /* Do actual hashing */ 886 /* Do actual hashing */
871#ifdef CONFIG_PPC_64K_PAGES 887#ifdef CONFIG_PPC_64K_PAGES
872 /* If _PAGE_4K_PFN is set, make sure this is a 4k segment */ 888 /* If _PAGE_4K_PFN is set, make sure this is a 4k segment */
873 if (pte_val(*ptep) & _PAGE_4K_PFN) { 889 if ((pte_val(*ptep) & _PAGE_4K_PFN) && psize == MMU_PAGE_64K) {
874 demote_segment_4k(mm, ea); 890 demote_segment_4k(mm, ea);
875 psize = MMU_PAGE_4K; 891 psize = MMU_PAGE_4K;
876 } 892 }
@@ -899,7 +915,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
899 } 915 }
900 } 916 }
901 if (user_region) { 917 if (user_region) {
902 if (psize != get_paca()->context.user_psize) { 918 if (psize != get_paca_psize(ea)) {
903 get_paca()->context = mm->context; 919 get_paca()->context = mm->context;
904 slb_flush_and_rebolt(); 920 slb_flush_and_rebolt();
905 } 921 }
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index a02266dad215..0d12fba31bc5 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -458,8 +458,7 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access,
458 old_pte = pte_val(*ptep); 458 old_pte = pte_val(*ptep);
459 if (old_pte & _PAGE_BUSY) 459 if (old_pte & _PAGE_BUSY)
460 goto out; 460 goto out;
461 new_pte = old_pte | _PAGE_BUSY | 461 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
462 _PAGE_ACCESSED | _PAGE_HASHPTE;
463 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep, 462 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
464 old_pte, new_pte)); 463 old_pte, new_pte));
465 464
@@ -499,12 +498,14 @@ repeat:
499 HPTES_PER_GROUP) & ~0x7UL; 498 HPTES_PER_GROUP) & ~0x7UL;
500 499
501 /* clear HPTE slot informations in new PTE */ 500 /* clear HPTE slot informations in new PTE */
501#ifdef CONFIG_PPC_64K_PAGES
502 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
503#else
502 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE; 504 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
503 505#endif
504 /* Add in WIMG bits */ 506 /* Add in WIMG bits */
505 /* XXX We should store these in the pte */ 507 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
506 /* --BenH: I think they are ... */ 508 _PAGE_COHERENT | _PAGE_GUARDED));
507 rflags |= _PAGE_COHERENT;
508 509
509 /* Insert into the hash table, primary slot */ 510 /* Insert into the hash table, primary slot */
510 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, 511 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c
index 1952b4d3fa7f..388ceda632f3 100644
--- a/arch/powerpc/mm/init_32.c
+++ b/arch/powerpc/mm/init_32.c
@@ -43,6 +43,7 @@
43#include <asm/btext.h> 43#include <asm/btext.h>
44#include <asm/tlb.h> 44#include <asm/tlb.h>
45#include <asm/sections.h> 45#include <asm/sections.h>
46#include <asm/system.h>
46 47
47#include "mmu_decl.h" 48#include "mmu_decl.h"
48 49
@@ -56,8 +57,8 @@
56 57
57DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 58DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
58 59
59unsigned long total_memory; 60phys_addr_t total_memory;
60unsigned long total_lowmem; 61phys_addr_t total_lowmem;
61 62
62phys_addr_t memstart_addr = (phys_addr_t)~0ull; 63phys_addr_t memstart_addr = (phys_addr_t)~0ull;
63EXPORT_SYMBOL(memstart_addr); 64EXPORT_SYMBOL(memstart_addr);
@@ -76,8 +77,6 @@ void MMU_init(void);
76/* XXX should be in current.h -- paulus */ 77/* XXX should be in current.h -- paulus */
77extern struct task_struct *current_set[NR_CPUS]; 78extern struct task_struct *current_set[NR_CPUS];
78 79
79extern int init_bootmem_done;
80
81/* 80/*
82 * this tells the system to map all of ram with the segregs 81 * this tells the system to map all of ram with the segregs
83 * (i.e. page tables) instead of the bats. 82 * (i.e. page tables) instead of the bats.
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index 6aa65375abf5..6ef63caca682 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -185,7 +185,7 @@ void pgtable_cache_init(void)
185 * do this by hand as the proffered address may not be correctly aligned. 185 * do this by hand as the proffered address may not be correctly aligned.
186 * Subtraction of non-aligned pointers produces undefined results. 186 * Subtraction of non-aligned pointers produces undefined results.
187 */ 187 */
188unsigned long __meminit vmemmap_section_start(unsigned long page) 188static unsigned long __meminit vmemmap_section_start(unsigned long page)
189{ 189{
190 unsigned long offset = page - ((unsigned long)(vmemmap)); 190 unsigned long offset = page - ((unsigned long)(vmemmap));
191 191
@@ -198,7 +198,7 @@ unsigned long __meminit vmemmap_section_start(unsigned long page)
198 * which overlaps this vmemmap page is initialised then this page is 198 * which overlaps this vmemmap page is initialised then this page is
199 * initialised already. 199 * initialised already.
200 */ 200 */
201int __meminit vmemmap_populated(unsigned long start, int page_size) 201static int __meminit vmemmap_populated(unsigned long start, int page_size)
202{ 202{
203 unsigned long end = start + page_size; 203 unsigned long end = start + page_size;
204 204
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 51f82d83bf14..1ca2235f0965 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -44,6 +44,7 @@
44#include <asm/btext.h> 44#include <asm/btext.h>
45#include <asm/tlb.h> 45#include <asm/tlb.h>
46#include <asm/sections.h> 46#include <asm/sections.h>
47#include <asm/sparsemem.h>
47#include <asm/vdso.h> 48#include <asm/vdso.h>
48#include <asm/fixmap.h> 49#include <asm/fixmap.h>
49 50
@@ -329,7 +330,7 @@ static int __init mark_nonram_nosave(void)
329void __init paging_init(void) 330void __init paging_init(void)
330{ 331{
331 unsigned long total_ram = lmb_phys_mem_size(); 332 unsigned long total_ram = lmb_phys_mem_size();
332 unsigned long top_of_ram = lmb_end_of_DRAM(); 333 phys_addr_t top_of_ram = lmb_end_of_DRAM();
333 unsigned long max_zone_pfns[MAX_NR_ZONES]; 334 unsigned long max_zone_pfns[MAX_NR_ZONES];
334 335
335#ifdef CONFIG_PPC32 336#ifdef CONFIG_PPC32
@@ -348,10 +349,10 @@ void __init paging_init(void)
348 kmap_prot = PAGE_KERNEL; 349 kmap_prot = PAGE_KERNEL;
349#endif /* CONFIG_HIGHMEM */ 350#endif /* CONFIG_HIGHMEM */
350 351
351 printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", 352 printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%lx\n",
352 top_of_ram, total_ram); 353 (u64)top_of_ram, total_ram);
353 printk(KERN_DEBUG "Memory hole size: %ldMB\n", 354 printk(KERN_DEBUG "Memory hole size: %ldMB\n",
354 (top_of_ram - total_ram) >> 20); 355 (long int)((top_of_ram - total_ram) >> 20));
355 memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 356 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
356#ifdef CONFIG_HIGHMEM 357#ifdef CONFIG_HIGHMEM
357 max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT; 358 max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h
index 04802252a64f..fab3cfad4099 100644
--- a/arch/powerpc/mm/mmu_decl.h
+++ b/arch/powerpc/mm/mmu_decl.h
@@ -29,7 +29,7 @@ extern void hash_preload(struct mm_struct *mm, unsigned long ea,
29#ifdef CONFIG_PPC32 29#ifdef CONFIG_PPC32
30extern void mapin_ram(void); 30extern void mapin_ram(void);
31extern int map_page(unsigned long va, phys_addr_t pa, int flags); 31extern int map_page(unsigned long va, phys_addr_t pa, int flags);
32extern void setbat(int index, unsigned long virt, unsigned long phys, 32extern void setbat(int index, unsigned long virt, phys_addr_t phys,
33 unsigned int size, int flags); 33 unsigned int size, int flags);
34extern void settlbcam(int index, unsigned long virt, phys_addr_t phys, 34extern void settlbcam(int index, unsigned long virt, phys_addr_t phys,
35 unsigned int size, int flags, unsigned int pid); 35 unsigned int size, int flags, unsigned int pid);
@@ -49,8 +49,8 @@ extern unsigned int num_tlbcam_entries;
49extern unsigned long ioremap_bot; 49extern unsigned long ioremap_bot;
50extern unsigned long __max_low_memory; 50extern unsigned long __max_low_memory;
51extern phys_addr_t __initial_memory_limit_addr; 51extern phys_addr_t __initial_memory_limit_addr;
52extern unsigned long total_memory; 52extern phys_addr_t total_memory;
53extern unsigned long total_lowmem; 53extern phys_addr_t total_lowmem;
54extern phys_addr_t memstart_addr; 54extern phys_addr_t memstart_addr;
55extern phys_addr_t lowmem_end_addr; 55extern phys_addr_t lowmem_end_addr;
56 56
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index dc704da363eb..cf4bffba6f7c 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -268,6 +268,144 @@ static unsigned long __devinit read_n_cells(int n, const unsigned int **buf)
268 return result; 268 return result;
269} 269}
270 270
271struct of_drconf_cell {
272 u64 base_addr;
273 u32 drc_index;
274 u32 reserved;
275 u32 aa_index;
276 u32 flags;
277};
278
279#define DRCONF_MEM_ASSIGNED 0x00000008
280#define DRCONF_MEM_AI_INVALID 0x00000040
281#define DRCONF_MEM_RESERVED 0x00000080
282
283/*
284 * Read the next lmb list entry from the ibm,dynamic-memory property
285 * and return the information in the provided of_drconf_cell structure.
286 */
287static void read_drconf_cell(struct of_drconf_cell *drmem, const u32 **cellp)
288{
289 const u32 *cp;
290
291 drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp);
292
293 cp = *cellp;
294 drmem->drc_index = cp[0];
295 drmem->reserved = cp[1];
296 drmem->aa_index = cp[2];
297 drmem->flags = cp[3];
298
299 *cellp = cp + 4;
300}
301
302/*
303 * Retreive and validate the ibm,dynamic-memory property of the device tree.
304 *
305 * The layout of the ibm,dynamic-memory property is a number N of lmb
306 * list entries followed by N lmb list entries. Each lmb list entry
307 * contains information as layed out in the of_drconf_cell struct above.
308 */
309static int of_get_drconf_memory(struct device_node *memory, const u32 **dm)
310{
311 const u32 *prop;
312 u32 len, entries;
313
314 prop = of_get_property(memory, "ibm,dynamic-memory", &len);
315 if (!prop || len < sizeof(unsigned int))
316 return 0;
317
318 entries = *prop++;
319
320 /* Now that we know the number of entries, revalidate the size
321 * of the property read in to ensure we have everything
322 */
323 if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int))
324 return 0;
325
326 *dm = prop;
327 return entries;
328}
329
330/*
331 * Retreive and validate the ibm,lmb-size property for drconf memory
332 * from the device tree.
333 */
334static u64 of_get_lmb_size(struct device_node *memory)
335{
336 const u32 *prop;
337 u32 len;
338
339 prop = of_get_property(memory, "ibm,lmb-size", &len);
340 if (!prop || len < sizeof(unsigned int))
341 return 0;
342
343 return read_n_cells(n_mem_size_cells, &prop);
344}
345
346struct assoc_arrays {
347 u32 n_arrays;
348 u32 array_sz;
349 const u32 *arrays;
350};
351
352/*
353 * Retreive and validate the list of associativity arrays for drconf
354 * memory from the ibm,associativity-lookup-arrays property of the
355 * device tree..
356 *
357 * The layout of the ibm,associativity-lookup-arrays property is a number N
358 * indicating the number of associativity arrays, followed by a number M
359 * indicating the size of each associativity array, followed by a list
360 * of N associativity arrays.
361 */
362static int of_get_assoc_arrays(struct device_node *memory,
363 struct assoc_arrays *aa)
364{
365 const u32 *prop;
366 u32 len;
367
368 prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len);
369 if (!prop || len < 2 * sizeof(unsigned int))
370 return -1;
371
372 aa->n_arrays = *prop++;
373 aa->array_sz = *prop++;
374
375 /* Now that we know the number of arrrays and size of each array,
376 * revalidate the size of the property read in.
377 */
378 if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int))
379 return -1;
380
381 aa->arrays = prop;
382 return 0;
383}
384
385/*
386 * This is like of_node_to_nid_single() for memory represented in the
387 * ibm,dynamic-reconfiguration-memory node.
388 */
389static int of_drconf_to_nid_single(struct of_drconf_cell *drmem,
390 struct assoc_arrays *aa)
391{
392 int default_nid = 0;
393 int nid = default_nid;
394 int index;
395
396 if (min_common_depth > 0 && min_common_depth <= aa->array_sz &&
397 !(drmem->flags & DRCONF_MEM_AI_INVALID) &&
398 drmem->aa_index < aa->n_arrays) {
399 index = drmem->aa_index * aa->array_sz + min_common_depth - 1;
400 nid = aa->arrays[index];
401
402 if (nid == 0xffff || nid >= MAX_NUMNODES)
403 nid = default_nid;
404 }
405
406 return nid;
407}
408
271/* 409/*
272 * Figure out to which domain a cpu belongs and stick it there. 410 * Figure out to which domain a cpu belongs and stick it there.
273 * Return the id of the domain used. 411 * Return the id of the domain used.
@@ -355,57 +493,50 @@ static unsigned long __init numa_enforce_memory_limit(unsigned long start,
355 */ 493 */
356static void __init parse_drconf_memory(struct device_node *memory) 494static void __init parse_drconf_memory(struct device_node *memory)
357{ 495{
358 const unsigned int *lm, *dm, *aa; 496 const u32 *dm;
359 unsigned int ls, ld, la; 497 unsigned int n, rc;
360 unsigned int n, aam, aalen; 498 unsigned long lmb_size, size;
361 unsigned long lmb_size, size, start; 499 int nid;
362 int nid, default_nid = 0; 500 struct assoc_arrays aa;
363 unsigned int ai, flags; 501
364 502 n = of_get_drconf_memory(memory, &dm);
365 lm = of_get_property(memory, "ibm,lmb-size", &ls); 503 if (!n)
366 dm = of_get_property(memory, "ibm,dynamic-memory", &ld); 504 return;
367 aa = of_get_property(memory, "ibm,associativity-lookup-arrays", &la); 505
368 if (!lm || !dm || !aa || 506 lmb_size = of_get_lmb_size(memory);
369 ls < sizeof(unsigned int) || ld < sizeof(unsigned int) || 507 if (!lmb_size)
370 la < 2 * sizeof(unsigned int))
371 return; 508 return;
372 509
373 lmb_size = read_n_cells(n_mem_size_cells, &lm); 510 rc = of_get_assoc_arrays(memory, &aa);
374 n = *dm++; /* number of LMBs */ 511 if (rc)
375 aam = *aa++; /* number of associativity lists */
376 aalen = *aa++; /* length of each associativity list */
377 if (ld < (n * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int) ||
378 la < (aam * aalen + 2) * sizeof(unsigned int))
379 return; 512 return;
380 513
381 for (; n != 0; --n) { 514 for (; n != 0; --n) {
382 start = read_n_cells(n_mem_addr_cells, &dm); 515 struct of_drconf_cell drmem;
383 ai = dm[2]; 516
384 flags = dm[3]; 517 read_drconf_cell(&drmem, &dm);
385 dm += 4; 518
386 /* 0x80 == reserved, 0x8 = assigned to us */ 519 /* skip this block if the reserved bit is set in flags (0x80)
387 if ((flags & 0x80) || !(flags & 0x8)) 520 or if the block is not assigned to this partition (0x8) */
521 if ((drmem.flags & DRCONF_MEM_RESERVED)
522 || !(drmem.flags & DRCONF_MEM_ASSIGNED))
388 continue; 523 continue;
389 nid = default_nid;
390 /* flags & 0x40 means associativity index is invalid */
391 if (min_common_depth > 0 && min_common_depth <= aalen &&
392 (flags & 0x40) == 0 && ai < aam) {
393 /* this is like of_node_to_nid_single */
394 nid = aa[ai * aalen + min_common_depth - 1];
395 if (nid == 0xffff || nid >= MAX_NUMNODES)
396 nid = default_nid;
397 }
398 524
399 fake_numa_create_new_node(((start + lmb_size) >> PAGE_SHIFT), 525 nid = of_drconf_to_nid_single(&drmem, &aa);
400 &nid); 526
527 fake_numa_create_new_node(
528 ((drmem.base_addr + lmb_size) >> PAGE_SHIFT),
529 &nid);
530
401 node_set_online(nid); 531 node_set_online(nid);
402 532
403 size = numa_enforce_memory_limit(start, lmb_size); 533 size = numa_enforce_memory_limit(drmem.base_addr, lmb_size);
404 if (!size) 534 if (!size)
405 continue; 535 continue;
406 536
407 add_active_range(nid, start >> PAGE_SHIFT, 537 add_active_range(nid, drmem.base_addr >> PAGE_SHIFT,
408 (start >> PAGE_SHIFT) + (size >> PAGE_SHIFT)); 538 (drmem.base_addr >> PAGE_SHIFT)
539 + (size >> PAGE_SHIFT));
409 } 540 }
410} 541}
411 542
@@ -770,6 +901,79 @@ early_param("numa", early_numa);
770 901
771#ifdef CONFIG_MEMORY_HOTPLUG 902#ifdef CONFIG_MEMORY_HOTPLUG
772/* 903/*
904 * Validate the node associated with the memory section we are
905 * trying to add.
906 */
907int valid_hot_add_scn(int *nid, unsigned long start, u32 lmb_size,
908 unsigned long scn_addr)
909{
910 nodemask_t nodes;
911
912 if (*nid < 0 || !node_online(*nid))
913 *nid = any_online_node(NODE_MASK_ALL);
914
915 if ((scn_addr >= start) && (scn_addr < (start + lmb_size))) {
916 nodes_setall(nodes);
917 while (NODE_DATA(*nid)->node_spanned_pages == 0) {
918 node_clear(*nid, nodes);
919 *nid = any_online_node(nodes);
920 }
921
922 return 1;
923 }
924
925 return 0;
926}
927
928/*
929 * Find the node associated with a hot added memory section represented
930 * by the ibm,dynamic-reconfiguration-memory node.
931 */
932static int hot_add_drconf_scn_to_nid(struct device_node *memory,
933 unsigned long scn_addr)
934{
935 const u32 *dm;
936 unsigned int n, rc;
937 unsigned long lmb_size;
938 int default_nid = any_online_node(NODE_MASK_ALL);
939 int nid;
940 struct assoc_arrays aa;
941
942 n = of_get_drconf_memory(memory, &dm);
943 if (!n)
944 return default_nid;;
945
946 lmb_size = of_get_lmb_size(memory);
947 if (!lmb_size)
948 return default_nid;
949
950 rc = of_get_assoc_arrays(memory, &aa);
951 if (rc)
952 return default_nid;
953
954 for (; n != 0; --n) {
955 struct of_drconf_cell drmem;
956
957 read_drconf_cell(&drmem, &dm);
958
959 /* skip this block if it is reserved or not assigned to
960 * this partition */
961 if ((drmem.flags & DRCONF_MEM_RESERVED)
962 || !(drmem.flags & DRCONF_MEM_ASSIGNED))
963 continue;
964
965 nid = of_drconf_to_nid_single(&drmem, &aa);
966
967 if (valid_hot_add_scn(&nid, drmem.base_addr, lmb_size,
968 scn_addr))
969 return nid;
970 }
971
972 BUG(); /* section address should be found above */
973 return 0;
974}
975
976/*
773 * Find the node associated with a hot added memory section. Section 977 * Find the node associated with a hot added memory section. Section
774 * corresponds to a SPARSEMEM section, not an LMB. It is assumed that 978 * corresponds to a SPARSEMEM section, not an LMB. It is assumed that
775 * sections are fully contained within a single LMB. 979 * sections are fully contained within a single LMB.
@@ -777,12 +981,17 @@ early_param("numa", early_numa);
777int hot_add_scn_to_nid(unsigned long scn_addr) 981int hot_add_scn_to_nid(unsigned long scn_addr)
778{ 982{
779 struct device_node *memory = NULL; 983 struct device_node *memory = NULL;
780 nodemask_t nodes;
781 int default_nid = any_online_node(NODE_MASK_ALL);
782 int nid; 984 int nid;
783 985
784 if (!numa_enabled || (min_common_depth < 0)) 986 if (!numa_enabled || (min_common_depth < 0))
785 return default_nid; 987 return any_online_node(NODE_MASK_ALL);
988
989 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
990 if (memory) {
991 nid = hot_add_drconf_scn_to_nid(memory, scn_addr);
992 of_node_put(memory);
993 return nid;
994 }
786 995
787 while ((memory = of_find_node_by_type(memory, "memory")) != NULL) { 996 while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
788 unsigned long start, size; 997 unsigned long start, size;
@@ -801,13 +1010,9 @@ ha_new_range:
801 size = read_n_cells(n_mem_size_cells, &memcell_buf); 1010 size = read_n_cells(n_mem_size_cells, &memcell_buf);
802 nid = of_node_to_nid_single(memory); 1011 nid = of_node_to_nid_single(memory);
803 1012
804 /* Domains not present at boot default to 0 */ 1013 if (valid_hot_add_scn(&nid, start, size, scn_addr)) {
805 if (nid < 0 || !node_online(nid))
806 nid = default_nid;
807
808 if ((scn_addr >= start) && (scn_addr < (start + size))) {
809 of_node_put(memory); 1014 of_node_put(memory);
810 goto got_nid; 1015 return nid;
811 } 1016 }
812 1017
813 if (--ranges) /* process all ranges in cell */ 1018 if (--ranges) /* process all ranges in cell */
@@ -815,14 +1020,5 @@ ha_new_range:
815 } 1020 }
816 BUG(); /* section address should be found above */ 1021 BUG(); /* section address should be found above */
817 return 0; 1022 return 0;
818
819 /* Temporary code to ensure that returned node is not empty */
820got_nid:
821 nodes_setall(nodes);
822 while (NODE_DATA(nid)->node_spanned_pages == 0) {
823 node_clear(nid, nodes);
824 nid = any_online_node(nodes);
825 }
826 return nid;
827} 1023}
828#endif /* CONFIG_MEMORY_HOTPLUG */ 1024#endif /* CONFIG_MEMORY_HOTPLUG */
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index e0ff59f21135..c7584072dfcc 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -53,9 +53,9 @@ extern void hash_page_sync(void);
53#endif 53#endif
54 54
55#ifdef HAVE_BATS 55#ifdef HAVE_BATS
56extern unsigned long v_mapped_by_bats(unsigned long va); 56extern phys_addr_t v_mapped_by_bats(unsigned long va);
57extern unsigned long p_mapped_by_bats(unsigned long pa); 57extern unsigned long p_mapped_by_bats(phys_addr_t pa);
58void setbat(int index, unsigned long virt, unsigned long phys, 58void setbat(int index, unsigned long virt, phys_addr_t phys,
59 unsigned int size, int flags); 59 unsigned int size, int flags);
60 60
61#else /* !HAVE_BATS */ 61#else /* !HAVE_BATS */
diff --git a/arch/powerpc/mm/ppc_mmu_32.c b/arch/powerpc/mm/ppc_mmu_32.c
index cef9f156874b..c53145f61942 100644
--- a/arch/powerpc/mm/ppc_mmu_32.c
+++ b/arch/powerpc/mm/ppc_mmu_32.c
@@ -38,21 +38,18 @@ struct hash_pte *Hash, *Hash_end;
38unsigned long Hash_size, Hash_mask; 38unsigned long Hash_size, Hash_mask;
39unsigned long _SDR1; 39unsigned long _SDR1;
40 40
41union ubat { /* BAT register values to be loaded */ 41struct ppc_bat BATS[8][2]; /* 8 pairs of IBAT, DBAT */
42 struct ppc_bat bat;
43 u32 word[2];
44} BATS[8][2]; /* 8 pairs of IBAT, DBAT */
45 42
46struct batrange { /* stores address ranges mapped by BATs */ 43struct batrange { /* stores address ranges mapped by BATs */
47 unsigned long start; 44 unsigned long start;
48 unsigned long limit; 45 unsigned long limit;
49 unsigned long phys; 46 phys_addr_t phys;
50} bat_addrs[8]; 47} bat_addrs[8];
51 48
52/* 49/*
53 * Return PA for this VA if it is mapped by a BAT, or 0 50 * Return PA for this VA if it is mapped by a BAT, or 0
54 */ 51 */
55unsigned long v_mapped_by_bats(unsigned long va) 52phys_addr_t v_mapped_by_bats(unsigned long va)
56{ 53{
57 int b; 54 int b;
58 for (b = 0; b < 4; ++b) 55 for (b = 0; b < 4; ++b)
@@ -64,7 +61,7 @@ unsigned long v_mapped_by_bats(unsigned long va)
64/* 61/*
65 * Return VA for a given PA or 0 if not mapped 62 * Return VA for a given PA or 0 if not mapped
66 */ 63 */
67unsigned long p_mapped_by_bats(unsigned long pa) 64unsigned long p_mapped_by_bats(phys_addr_t pa)
68{ 65{
69 int b; 66 int b;
70 for (b = 0; b < 4; ++b) 67 for (b = 0; b < 4; ++b)
@@ -119,12 +116,12 @@ unsigned long __init mmu_mapin_ram(void)
119 * The parameters are not checked; in particular size must be a power 116 * The parameters are not checked; in particular size must be a power
120 * of 2 between 128k and 256M. 117 * of 2 between 128k and 256M.
121 */ 118 */
122void __init setbat(int index, unsigned long virt, unsigned long phys, 119void __init setbat(int index, unsigned long virt, phys_addr_t phys,
123 unsigned int size, int flags) 120 unsigned int size, int flags)
124{ 121{
125 unsigned int bl; 122 unsigned int bl;
126 int wimgxpp; 123 int wimgxpp;
127 union ubat *bat = BATS[index]; 124 struct ppc_bat *bat = BATS[index];
128 125
129 if (((flags & _PAGE_NO_CACHE) == 0) && 126 if (((flags & _PAGE_NO_CACHE) == 0) &&
130 cpu_has_feature(CPU_FTR_NEED_COHERENT)) 127 cpu_has_feature(CPU_FTR_NEED_COHERENT))
@@ -137,15 +134,15 @@ void __init setbat(int index, unsigned long virt, unsigned long phys,
137 wimgxpp = flags & (_PAGE_WRITETHRU | _PAGE_NO_CACHE 134 wimgxpp = flags & (_PAGE_WRITETHRU | _PAGE_NO_CACHE
138 | _PAGE_COHERENT | _PAGE_GUARDED); 135 | _PAGE_COHERENT | _PAGE_GUARDED);
139 wimgxpp |= (flags & _PAGE_RW)? BPP_RW: BPP_RX; 136 wimgxpp |= (flags & _PAGE_RW)? BPP_RW: BPP_RX;
140 bat[1].word[0] = virt | (bl << 2) | 2; /* Vs=1, Vp=0 */ 137 bat[1].batu = virt | (bl << 2) | 2; /* Vs=1, Vp=0 */
141 bat[1].word[1] = phys | wimgxpp; 138 bat[1].batl = BAT_PHYS_ADDR(phys) | wimgxpp;
142#ifndef CONFIG_KGDB /* want user access for breakpoints */ 139#ifndef CONFIG_KGDB /* want user access for breakpoints */
143 if (flags & _PAGE_USER) 140 if (flags & _PAGE_USER)
144#endif 141#endif
145 bat[1].bat.batu.vp = 1; 142 bat[1].batu |= 1; /* Vp = 1 */
146 if (flags & _PAGE_GUARDED) { 143 if (flags & _PAGE_GUARDED) {
147 /* G bit must be zero in IBATs */ 144 /* G bit must be zero in IBATs */
148 bat[0].word[0] = bat[0].word[1] = 0; 145 bat[0].batu = bat[0].batl = 0;
149 } else { 146 } else {
150 /* make IBAT same as DBAT */ 147 /* make IBAT same as DBAT */
151 bat[0] = bat[1]; 148 bat[0] = bat[1];
@@ -158,8 +155,8 @@ void __init setbat(int index, unsigned long virt, unsigned long phys,
158 | _PAGE_COHERENT); 155 | _PAGE_COHERENT);
159 wimgxpp |= (flags & _PAGE_RW)? 156 wimgxpp |= (flags & _PAGE_RW)?
160 ((flags & _PAGE_USER)? PP_RWRW: PP_RWXX): PP_RXRX; 157 ((flags & _PAGE_USER)? PP_RWRW: PP_RWXX): PP_RXRX;
161 bat->word[0] = virt | wimgxpp | 4; /* Ks=0, Ku=1 */ 158 bat->batu = virt | wimgxpp | 4; /* Ks=0, Ku=1 */
162 bat->word[1] = phys | bl | 0x40; /* V=1 */ 159 bat->batl = phys | bl | 0x40; /* V=1 */
163 } 160 }
164 161
165 bat_addrs[index].start = virt; 162 bat_addrs[index].start = virt;
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index 2bd12d965db1..db44e02e045b 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -215,10 +215,7 @@ static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psiz
215 mm->context.high_slices_psize); 215 mm->context.high_slices_psize);
216 216
217 spin_unlock_irqrestore(&slice_convert_lock, flags); 217 spin_unlock_irqrestore(&slice_convert_lock, flags);
218 mb();
219 218
220 /* XXX this is sub-optimal but will do for now */
221 on_each_cpu(slice_flush_segments, mm, 1);
222#ifdef CONFIG_SPU_BASE 219#ifdef CONFIG_SPU_BASE
223 spu_flush_all_slbs(mm); 220 spu_flush_all_slbs(mm);
224#endif 221#endif
@@ -384,17 +381,34 @@ static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
384 return slice_find_area_bottomup(mm, len, mask, psize, use_cache); 381 return slice_find_area_bottomup(mm, len, mask, psize, use_cache);
385} 382}
386 383
384#define or_mask(dst, src) do { \
385 (dst).low_slices |= (src).low_slices; \
386 (dst).high_slices |= (src).high_slices; \
387} while (0)
388
389#define andnot_mask(dst, src) do { \
390 (dst).low_slices &= ~(src).low_slices; \
391 (dst).high_slices &= ~(src).high_slices; \
392} while (0)
393
394#ifdef CONFIG_PPC_64K_PAGES
395#define MMU_PAGE_BASE MMU_PAGE_64K
396#else
397#define MMU_PAGE_BASE MMU_PAGE_4K
398#endif
399
387unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, 400unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
388 unsigned long flags, unsigned int psize, 401 unsigned long flags, unsigned int psize,
389 int topdown, int use_cache) 402 int topdown, int use_cache)
390{ 403{
391 struct slice_mask mask; 404 struct slice_mask mask = {0, 0};
392 struct slice_mask good_mask; 405 struct slice_mask good_mask;
393 struct slice_mask potential_mask = {0,0} /* silence stupid warning */; 406 struct slice_mask potential_mask = {0,0} /* silence stupid warning */;
394 int pmask_set = 0; 407 struct slice_mask compat_mask = {0, 0};
395 int fixed = (flags & MAP_FIXED); 408 int fixed = (flags & MAP_FIXED);
396 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); 409 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
397 struct mm_struct *mm = current->mm; 410 struct mm_struct *mm = current->mm;
411 unsigned long newaddr;
398 412
399 /* Sanity checks */ 413 /* Sanity checks */
400 BUG_ON(mm->task_size == 0); 414 BUG_ON(mm->task_size == 0);
@@ -416,21 +430,48 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
416 if (!fixed && addr) { 430 if (!fixed && addr) {
417 addr = _ALIGN_UP(addr, 1ul << pshift); 431 addr = _ALIGN_UP(addr, 1ul << pshift);
418 slice_dbg(" aligned addr=%lx\n", addr); 432 slice_dbg(" aligned addr=%lx\n", addr);
433 /* Ignore hint if it's too large or overlaps a VMA */
434 if (addr > mm->task_size - len ||
435 !slice_area_is_free(mm, addr, len))
436 addr = 0;
419 } 437 }
420 438
421 /* First makeup a "good" mask of slices that have the right size 439 /* First make up a "good" mask of slices that have the right size
422 * already 440 * already
423 */ 441 */
424 good_mask = slice_mask_for_size(mm, psize); 442 good_mask = slice_mask_for_size(mm, psize);
425 slice_print_mask(" good_mask", good_mask); 443 slice_print_mask(" good_mask", good_mask);
426 444
427 /* First check hint if it's valid or if we have MAP_FIXED */ 445 /*
428 if ((addr != 0 || fixed) && (mm->task_size - len) >= addr) { 446 * Here "good" means slices that are already the right page size,
447 * "compat" means slices that have a compatible page size (i.e.
448 * 4k in a 64k pagesize kernel), and "free" means slices without
449 * any VMAs.
450 *
451 * If MAP_FIXED:
452 * check if fits in good | compat => OK
453 * check if fits in good | compat | free => convert free
454 * else bad
455 * If have hint:
456 * check if hint fits in good => OK
457 * check if hint fits in good | free => convert free
458 * Otherwise:
459 * search in good, found => OK
460 * search in good | free, found => convert free
461 * search in good | compat | free, found => convert free.
462 */
429 463
430 /* Don't bother with hint if it overlaps a VMA */ 464#ifdef CONFIG_PPC_64K_PAGES
431 if (!fixed && !slice_area_is_free(mm, addr, len)) 465 /* If we support combo pages, we can allow 64k pages in 4k slices */
432 goto search; 466 if (psize == MMU_PAGE_64K) {
467 compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K);
468 if (fixed)
469 or_mask(good_mask, compat_mask);
470 }
471#endif
433 472
473 /* First check hint if it's valid or if we have MAP_FIXED */
474 if (addr != 0 || fixed) {
434 /* Build a mask for the requested range */ 475 /* Build a mask for the requested range */
435 mask = slice_range_to_mask(addr, len); 476 mask = slice_range_to_mask(addr, len);
436 slice_print_mask(" mask", mask); 477 slice_print_mask(" mask", mask);
@@ -442,54 +483,66 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
442 slice_dbg(" fits good !\n"); 483 slice_dbg(" fits good !\n");
443 return addr; 484 return addr;
444 } 485 }
445 486 } else {
446 /* We don't fit in the good mask, check what other slices are 487 /* Now let's see if we can find something in the existing
447 * empty and thus can be converted 488 * slices for that size
448 */ 489 */
449 potential_mask = slice_mask_for_free(mm); 490 newaddr = slice_find_area(mm, len, good_mask, psize, topdown,
450 potential_mask.low_slices |= good_mask.low_slices; 491 use_cache);
451 potential_mask.high_slices |= good_mask.high_slices; 492 if (newaddr != -ENOMEM) {
452 pmask_set = 1; 493 /* Found within the good mask, we don't have to setup,
453 slice_print_mask(" potential", potential_mask); 494 * we thus return directly
454 if (slice_check_fit(mask, potential_mask)) { 495 */
455 slice_dbg(" fits potential !\n"); 496 slice_dbg(" found area at 0x%lx\n", newaddr);
456 goto convert; 497 return newaddr;
457 } 498 }
458 } 499 }
459 500
460 /* If we have MAP_FIXED and failed the above step, then error out */ 501 /* We don't fit in the good mask, check what other slices are
502 * empty and thus can be converted
503 */
504 potential_mask = slice_mask_for_free(mm);
505 or_mask(potential_mask, good_mask);
506 slice_print_mask(" potential", potential_mask);
507
508 if ((addr != 0 || fixed) && slice_check_fit(mask, potential_mask)) {
509 slice_dbg(" fits potential !\n");
510 goto convert;
511 }
512
513 /* If we have MAP_FIXED and failed the above steps, then error out */
461 if (fixed) 514 if (fixed)
462 return -EBUSY; 515 return -EBUSY;
463 516
464 search:
465 slice_dbg(" search...\n"); 517 slice_dbg(" search...\n");
466 518
467 /* Now let's see if we can find something in the existing slices 519 /* If we had a hint that didn't work out, see if we can fit
468 * for that size 520 * anywhere in the good area.
469 */ 521 */
470 addr = slice_find_area(mm, len, good_mask, psize, topdown, use_cache); 522 if (addr) {
471 if (addr != -ENOMEM) { 523 addr = slice_find_area(mm, len, good_mask, psize, topdown,
472 /* Found within the good mask, we don't have to setup, 524 use_cache);
473 * we thus return directly 525 if (addr != -ENOMEM) {
474 */ 526 slice_dbg(" found area at 0x%lx\n", addr);
475 slice_dbg(" found area at 0x%lx\n", addr); 527 return addr;
476 return addr; 528 }
477 }
478
479 /* Won't fit, check what can be converted */
480 if (!pmask_set) {
481 potential_mask = slice_mask_for_free(mm);
482 potential_mask.low_slices |= good_mask.low_slices;
483 potential_mask.high_slices |= good_mask.high_slices;
484 pmask_set = 1;
485 slice_print_mask(" potential", potential_mask);
486 } 529 }
487 530
488 /* Now let's see if we can find something in the existing slices 531 /* Now let's see if we can find something in the existing slices
489 * for that size 532 * for that size plus free slices
490 */ 533 */
491 addr = slice_find_area(mm, len, potential_mask, psize, topdown, 534 addr = slice_find_area(mm, len, potential_mask, psize, topdown,
492 use_cache); 535 use_cache);
536
537#ifdef CONFIG_PPC_64K_PAGES
538 if (addr == -ENOMEM && psize == MMU_PAGE_64K) {
539 /* retry the search with 4k-page slices included */
540 or_mask(potential_mask, compat_mask);
541 addr = slice_find_area(mm, len, potential_mask, psize,
542 topdown, use_cache);
543 }
544#endif
545
493 if (addr == -ENOMEM) 546 if (addr == -ENOMEM)
494 return -ENOMEM; 547 return -ENOMEM;
495 548
@@ -498,7 +551,13 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
498 slice_print_mask(" mask", mask); 551 slice_print_mask(" mask", mask);
499 552
500 convert: 553 convert:
501 slice_convert(mm, mask, psize); 554 andnot_mask(mask, good_mask);
555 andnot_mask(mask, compat_mask);
556 if (mask.low_slices || mask.high_slices) {
557 slice_convert(mm, mask, psize);
558 if (psize > MMU_PAGE_BASE)
559 on_each_cpu(slice_flush_segments, mm, 1);
560 }
502 return addr; 561 return addr;
503 562
504} 563}
@@ -598,6 +657,36 @@ void slice_set_user_psize(struct mm_struct *mm, unsigned int psize)
598 spin_unlock_irqrestore(&slice_convert_lock, flags); 657 spin_unlock_irqrestore(&slice_convert_lock, flags);
599} 658}
600 659
660void slice_set_psize(struct mm_struct *mm, unsigned long address,
661 unsigned int psize)
662{
663 unsigned long i, flags;
664 u64 *p;
665
666 spin_lock_irqsave(&slice_convert_lock, flags);
667 if (address < SLICE_LOW_TOP) {
668 i = GET_LOW_SLICE_INDEX(address);
669 p = &mm->context.low_slices_psize;
670 } else {
671 i = GET_HIGH_SLICE_INDEX(address);
672 p = &mm->context.high_slices_psize;
673 }
674 *p = (*p & ~(0xful << (i * 4))) | ((unsigned long) psize << (i * 4));
675 spin_unlock_irqrestore(&slice_convert_lock, flags);
676
677#ifdef CONFIG_SPU_BASE
678 spu_flush_all_slbs(mm);
679#endif
680}
681
682void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
683 unsigned long len, unsigned int psize)
684{
685 struct slice_mask mask = slice_range_to_mask(start, len);
686
687 slice_convert(mm, mask, psize);
688}
689
601/* 690/*
602 * is_hugepage_only_range() is used by generic code to verify wether 691 * is_hugepage_only_range() is used by generic code to verify wether
603 * a normal mmap mapping (non hugetlbfs) is valid on a given area. 692 * a normal mmap mapping (non hugetlbfs) is valid on a given area.
diff --git a/arch/powerpc/mm/stab.c b/arch/powerpc/mm/stab.c
index efbbd13d93e5..60e6032a8088 100644
--- a/arch/powerpc/mm/stab.c
+++ b/arch/powerpc/mm/stab.c
@@ -30,8 +30,8 @@ struct stab_entry {
30}; 30};
31 31
32#define NR_STAB_CACHE_ENTRIES 8 32#define NR_STAB_CACHE_ENTRIES 8
33DEFINE_PER_CPU(long, stab_cache_ptr); 33static DEFINE_PER_CPU(long, stab_cache_ptr);
34DEFINE_PER_CPU(long, stab_cache[NR_STAB_CACHE_ENTRIES]); 34static DEFINE_PER_CPU(long, stab_cache[NR_STAB_CACHE_ENTRIES]);
35 35
36/* 36/*
37 * Create a segment table entry for the given esid/vsid pair. 37 * Create a segment table entry for the given esid/vsid pair.
diff --git a/arch/powerpc/mm/tlb_64.c b/arch/powerpc/mm/tlb_64.c
index 69ad829a7fa3..a01b5c608ff9 100644
--- a/arch/powerpc/mm/tlb_64.c
+++ b/arch/powerpc/mm/tlb_64.c
@@ -37,8 +37,8 @@ DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
37 * include/asm-powerpc/tlb.h file -- tgall 37 * include/asm-powerpc/tlb.h file -- tgall
38 */ 38 */
39DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 39DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
40DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur); 40static DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
41unsigned long pte_freelist_forced_free; 41static unsigned long pte_freelist_forced_free;
42 42
43struct pte_freelist_batch 43struct pte_freelist_batch
44{ 44{
@@ -47,9 +47,6 @@ struct pte_freelist_batch
47 pgtable_free_t tables[0]; 47 pgtable_free_t tables[0];
48}; 48};
49 49
50DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
51unsigned long pte_freelist_forced_free;
52
53#define PTE_FREELIST_SIZE \ 50#define PTE_FREELIST_SIZE \
54 ((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \ 51 ((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \
55 / sizeof(pgtable_free_t)) 52 / sizeof(pgtable_free_t))
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-16 18:27:16 -0500