aboutsummaryrefslogtreecommitdiffstats
path: root/arch/ppc64/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/ppc64/mm')
-rw-r--r--arch/ppc64/mm/hash_low.S4
-rw-r--r--arch/ppc64/mm/hash_native.c3
-rw-r--r--arch/ppc64/mm/hash_utils.c4
-rw-r--r--arch/ppc64/mm/hugetlbpage.c388
-rw-r--r--arch/ppc64/mm/imalloc.c2
-rw-r--r--arch/ppc64/mm/init.c96
-rw-r--r--arch/ppc64/mm/numa.c2
-rw-r--r--arch/ppc64/mm/slb_low.S35
-rw-r--r--arch/ppc64/mm/tlb.c95
9 files changed, 343 insertions, 286 deletions
diff --git a/arch/ppc64/mm/hash_low.S b/arch/ppc64/mm/hash_low.S
index fbff24827ae7..35eb49e1b890 100644
--- a/arch/ppc64/mm/hash_low.S
+++ b/arch/ppc64/mm/hash_low.S
@@ -129,12 +129,10 @@ _GLOBAL(__hash_page)
129 * code rather than call a C function...) 129 * code rather than call a C function...)
130 */ 130 */
131BEGIN_FTR_SECTION 131BEGIN_FTR_SECTION
132BEGIN_FTR_SECTION
133 mr r4,r30 132 mr r4,r30
134 mr r5,r7 133 mr r5,r7
135 bl .hash_page_do_lazy_icache 134 bl .hash_page_do_lazy_icache
136END_FTR_SECTION_IFSET(CPU_FTR_NOEXECUTE) 135END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
137END_FTR_SECTION_IFCLR(CPU_FTR_COHERENT_ICACHE)
138 136
139 /* At this point, r3 contains new PP bits, save them in 137 /* At this point, r3 contains new PP bits, save them in
140 * place of "access" in the param area (sic) 138 * place of "access" in the param area (sic)
diff --git a/arch/ppc64/mm/hash_native.c b/arch/ppc64/mm/hash_native.c
index a6abd3a979bf..7626bb59954d 100644
--- a/arch/ppc64/mm/hash_native.c
+++ b/arch/ppc64/mm/hash_native.c
@@ -51,7 +51,6 @@ long native_hpte_insert(unsigned long hpte_group, unsigned long va,
51 unsigned long prpn, unsigned long vflags, 51 unsigned long prpn, unsigned long vflags,
52 unsigned long rflags) 52 unsigned long rflags)
53{ 53{
54 unsigned long arpn = physRpn_to_absRpn(prpn);
55 hpte_t *hptep = htab_address + hpte_group; 54 hpte_t *hptep = htab_address + hpte_group;
56 unsigned long hpte_v, hpte_r; 55 unsigned long hpte_v, hpte_r;
57 int i; 56 int i;
@@ -74,7 +73,7 @@ long native_hpte_insert(unsigned long hpte_group, unsigned long va,
74 hpte_v = (va >> 23) << HPTE_V_AVPN_SHIFT | vflags | HPTE_V_VALID; 73 hpte_v = (va >> 23) << HPTE_V_AVPN_SHIFT | vflags | HPTE_V_VALID;
75 if (vflags & HPTE_V_LARGE) 74 if (vflags & HPTE_V_LARGE)
76 va &= ~(1UL << HPTE_V_AVPN_SHIFT); 75 va &= ~(1UL << HPTE_V_AVPN_SHIFT);
77 hpte_r = (arpn << HPTE_R_RPN_SHIFT) | rflags; 76 hpte_r = (prpn << HPTE_R_RPN_SHIFT) | rflags;
78 77
79 hptep->r = hpte_r; 78 hptep->r = hpte_r;
80 /* Guarantee the second dword is visible before the valid bit */ 79 /* Guarantee the second dword is visible before the valid bit */
diff --git a/arch/ppc64/mm/hash_utils.c b/arch/ppc64/mm/hash_utils.c
index 623b5d130c31..09475c8edf7c 100644
--- a/arch/ppc64/mm/hash_utils.c
+++ b/arch/ppc64/mm/hash_utils.c
@@ -210,7 +210,7 @@ void __init htab_initialize(void)
210 210
211 /* create bolted the linear mapping in the hash table */ 211 /* create bolted the linear mapping in the hash table */
212 for (i=0; i < lmb.memory.cnt; i++) { 212 for (i=0; i < lmb.memory.cnt; i++) {
213 base = lmb.memory.region[i].physbase + KERNELBASE; 213 base = lmb.memory.region[i].base + KERNELBASE;
214 size = lmb.memory.region[i].size; 214 size = lmb.memory.region[i].size;
215 215
216 DBG("creating mapping for region: %lx : %lx\n", base, size); 216 DBG("creating mapping for region: %lx : %lx\n", base, size);
@@ -302,7 +302,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
302 int local = 0; 302 int local = 0;
303 cpumask_t tmp; 303 cpumask_t tmp;
304 304
305 if ((ea & ~REGION_MASK) > EADDR_MASK) 305 if ((ea & ~REGION_MASK) >= PGTABLE_RANGE)
306 return 1; 306 return 1;
307 307
308 switch (REGION_ID(ea)) { 308 switch (REGION_ID(ea)) {
diff --git a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c
index f9524602818d..e7833c80eb68 100644
--- a/arch/ppc64/mm/hugetlbpage.c
+++ b/arch/ppc64/mm/hugetlbpage.c
@@ -27,124 +27,94 @@
27 27
28#include <linux/sysctl.h> 28#include <linux/sysctl.h>
29 29
30#define HUGEPGDIR_SHIFT (HPAGE_SHIFT + PAGE_SHIFT - 3) 30#define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
31#define HUGEPGDIR_SIZE (1UL << HUGEPGDIR_SHIFT) 31#define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
32#define HUGEPGDIR_MASK (~(HUGEPGDIR_SIZE-1))
33 32
34#define HUGEPTE_INDEX_SIZE 9 33/* Modelled after find_linux_pte() */
35#define HUGEPGD_INDEX_SIZE 10 34pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
36
37#define PTRS_PER_HUGEPTE (1 << HUGEPTE_INDEX_SIZE)
38#define PTRS_PER_HUGEPGD (1 << HUGEPGD_INDEX_SIZE)
39
40static inline int hugepgd_index(unsigned long addr)
41{
42 return (addr & ~REGION_MASK) >> HUGEPGDIR_SHIFT;
43}
44
45static pud_t *hugepgd_offset(struct mm_struct *mm, unsigned long addr)
46{ 35{
47 int index; 36 pgd_t *pg;
37 pud_t *pu;
38 pmd_t *pm;
39 pte_t *pt;
48 40
49 if (! mm->context.huge_pgdir) 41 BUG_ON(! in_hugepage_area(mm->context, addr));
50 return NULL;
51 42
43 addr &= HPAGE_MASK;
44
45 pg = pgd_offset(mm, addr);
46 if (!pgd_none(*pg)) {
47 pu = pud_offset(pg, addr);
48 if (!pud_none(*pu)) {
49 pm = pmd_offset(pu, addr);
50 pt = (pte_t *)pm;
51 BUG_ON(!pmd_none(*pm)
52 && !(pte_present(*pt) && pte_huge(*pt)));
53 return pt;
54 }
55 }
52 56
53 index = hugepgd_index(addr); 57 return NULL;
54 BUG_ON(index >= PTRS_PER_HUGEPGD);
55 return (pud_t *)(mm->context.huge_pgdir + index);
56} 58}
57 59
58static inline pte_t *hugepte_offset(pud_t *dir, unsigned long addr) 60pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
59{ 61{
60 int index; 62 pgd_t *pg;
61 63 pud_t *pu;
62 if (pud_none(*dir)) 64 pmd_t *pm;
63 return NULL; 65 pte_t *pt;
64 66
65 index = (addr >> HPAGE_SHIFT) % PTRS_PER_HUGEPTE;
66 return (pte_t *)pud_page(*dir) + index;
67}
68
69static pud_t *hugepgd_alloc(struct mm_struct *mm, unsigned long addr)
70{
71 BUG_ON(! in_hugepage_area(mm->context, addr)); 67 BUG_ON(! in_hugepage_area(mm->context, addr));
72 68
73 if (! mm->context.huge_pgdir) { 69 addr &= HPAGE_MASK;
74 pgd_t *new;
75 spin_unlock(&mm->page_table_lock);
76 /* Don't use pgd_alloc(), because we want __GFP_REPEAT */
77 new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
78 BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
79 spin_lock(&mm->page_table_lock);
80 70
81 /* 71 pg = pgd_offset(mm, addr);
82 * Because we dropped the lock, we should re-check the 72 pu = pud_alloc(mm, pg, addr);
83 * entry, as somebody else could have populated it..
84 */
85 if (mm->context.huge_pgdir)
86 pgd_free(new);
87 else
88 mm->context.huge_pgdir = new;
89 }
90 return hugepgd_offset(mm, addr);
91}
92 73
93static pte_t *hugepte_alloc(struct mm_struct *mm, pud_t *dir, unsigned long addr) 74 if (pu) {
94{ 75 pm = pmd_alloc(mm, pu, addr);
95 if (! pud_present(*dir)) { 76 if (pm) {
96 pte_t *new; 77 pt = (pte_t *)pm;
97 78 BUG_ON(!pmd_none(*pm)
98 spin_unlock(&mm->page_table_lock); 79 && !(pte_present(*pt) && pte_huge(*pt)));
99 new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT); 80 return pt;
100 BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
101 spin_lock(&mm->page_table_lock);
102 /*
103 * Because we dropped the lock, we should re-check the
104 * entry, as somebody else could have populated it..
105 */
106 if (pud_present(*dir)) {
107 if (new)
108 kmem_cache_free(zero_cache, new);
109 } else {
110 struct page *ptepage;
111
112 if (! new)
113 return NULL;
114 ptepage = virt_to_page(new);
115 ptepage->mapping = (void *) mm;
116 ptepage->index = addr & HUGEPGDIR_MASK;
117 pud_populate(mm, dir, new);
118 } 81 }
119 } 82 }
120 83
121 return hugepte_offset(dir, addr); 84 return NULL;
122} 85}
123 86
124pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 87#define HUGEPTE_BATCH_SIZE (HPAGE_SIZE / PMD_SIZE)
125{
126 pud_t *pud;
127 88
128 BUG_ON(! in_hugepage_area(mm->context, addr)); 89void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
90 pte_t *ptep, pte_t pte)
91{
92 int i;
129 93
130 pud = hugepgd_offset(mm, addr); 94 if (pte_present(*ptep)) {
131 if (! pud) 95 pte_clear(mm, addr, ptep);
132 return NULL; 96 flush_tlb_pending();
97 }
133 98
134 return hugepte_offset(pud, addr); 99 for (i = 0; i < HUGEPTE_BATCH_SIZE; i++) {
100 *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
101 ptep++;
102 }
135} 103}
136 104
137pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) 105pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
106 pte_t *ptep)
138{ 107{
139 pud_t *pud; 108 unsigned long old = pte_update(ptep, ~0UL);
109 int i;
140 110
141 BUG_ON(! in_hugepage_area(mm->context, addr)); 111 if (old & _PAGE_HASHPTE)
112 hpte_update(mm, addr, old, 0);
142 113
143 pud = hugepgd_alloc(mm, addr); 114 for (i = 1; i < HUGEPTE_BATCH_SIZE; i++)
144 if (! pud) 115 ptep[i] = __pte(0);
145 return NULL;
146 116
147 return hugepte_alloc(mm, pud, addr); 117 return __pte(old);
148} 118}
149 119
150/* 120/*
@@ -162,15 +132,17 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
162 return 0; 132 return 0;
163} 133}
164 134
165static void flush_segments(void *parm) 135static void flush_low_segments(void *parm)
166{ 136{
167 u16 segs = (unsigned long) parm; 137 u16 areas = (unsigned long) parm;
168 unsigned long i; 138 unsigned long i;
169 139
170 asm volatile("isync" : : : "memory"); 140 asm volatile("isync" : : : "memory");
171 141
172 for (i = 0; i < 16; i++) { 142 BUILD_BUG_ON((sizeof(areas)*8) != NUM_LOW_AREAS);
173 if (! (segs & (1U << i))) 143
144 for (i = 0; i < NUM_LOW_AREAS; i++) {
145 if (! (areas & (1U << i)))
174 continue; 146 continue;
175 asm volatile("slbie %0" : : "r" (i << SID_SHIFT)); 147 asm volatile("slbie %0" : : "r" (i << SID_SHIFT));
176 } 148 }
@@ -178,13 +150,33 @@ static void flush_segments(void *parm)
178 asm volatile("isync" : : : "memory"); 150 asm volatile("isync" : : : "memory");
179} 151}
180 152
181static int prepare_low_seg_for_htlb(struct mm_struct *mm, unsigned long seg) 153static void flush_high_segments(void *parm)
182{ 154{
183 unsigned long start = seg << SID_SHIFT; 155 u16 areas = (unsigned long) parm;
184 unsigned long end = (seg+1) << SID_SHIFT; 156 unsigned long i, j;
157
158 asm volatile("isync" : : : "memory");
159
160 BUILD_BUG_ON((sizeof(areas)*8) != NUM_HIGH_AREAS);
161
162 for (i = 0; i < NUM_HIGH_AREAS; i++) {
163 if (! (areas & (1U << i)))
164 continue;
165 for (j = 0; j < (1UL << (HTLB_AREA_SHIFT-SID_SHIFT)); j++)
166 asm volatile("slbie %0"
167 :: "r" ((i << HTLB_AREA_SHIFT) + (j << SID_SHIFT)));
168 }
169
170 asm volatile("isync" : : : "memory");
171}
172
173static int prepare_low_area_for_htlb(struct mm_struct *mm, unsigned long area)
174{
175 unsigned long start = area << SID_SHIFT;
176 unsigned long end = (area+1) << SID_SHIFT;
185 struct vm_area_struct *vma; 177 struct vm_area_struct *vma;
186 178
187 BUG_ON(seg >= 16); 179 BUG_ON(area >= NUM_LOW_AREAS);
188 180
189 /* Check no VMAs are in the region */ 181 /* Check no VMAs are in the region */
190 vma = find_vma(mm, start); 182 vma = find_vma(mm, start);
@@ -194,20 +186,39 @@ static int prepare_low_seg_for_htlb(struct mm_struct *mm, unsigned long seg)
194 return 0; 186 return 0;
195} 187}
196 188
197static int open_low_hpage_segs(struct mm_struct *mm, u16 newsegs) 189static int prepare_high_area_for_htlb(struct mm_struct *mm, unsigned long area)
190{
191 unsigned long start = area << HTLB_AREA_SHIFT;
192 unsigned long end = (area+1) << HTLB_AREA_SHIFT;
193 struct vm_area_struct *vma;
194
195 BUG_ON(area >= NUM_HIGH_AREAS);
196
197 /* Check no VMAs are in the region */
198 vma = find_vma(mm, start);
199 if (vma && (vma->vm_start < end))
200 return -EBUSY;
201
202 return 0;
203}
204
205static int open_low_hpage_areas(struct mm_struct *mm, u16 newareas)
198{ 206{
199 unsigned long i; 207 unsigned long i;
200 208
201 newsegs &= ~(mm->context.htlb_segs); 209 BUILD_BUG_ON((sizeof(newareas)*8) != NUM_LOW_AREAS);
202 if (! newsegs) 210 BUILD_BUG_ON((sizeof(mm->context.low_htlb_areas)*8) != NUM_LOW_AREAS);
211
212 newareas &= ~(mm->context.low_htlb_areas);
213 if (! newareas)
203 return 0; /* The segments we want are already open */ 214 return 0; /* The segments we want are already open */
204 215
205 for (i = 0; i < 16; i++) 216 for (i = 0; i < NUM_LOW_AREAS; i++)
206 if ((1 << i) & newsegs) 217 if ((1 << i) & newareas)
207 if (prepare_low_seg_for_htlb(mm, i) != 0) 218 if (prepare_low_area_for_htlb(mm, i) != 0)
208 return -EBUSY; 219 return -EBUSY;
209 220
210 mm->context.htlb_segs |= newsegs; 221 mm->context.low_htlb_areas |= newareas;
211 222
212 /* update the paca copy of the context struct */ 223 /* update the paca copy of the context struct */
213 get_paca()->context = mm->context; 224 get_paca()->context = mm->context;
@@ -215,29 +226,63 @@ static int open_low_hpage_segs(struct mm_struct *mm, u16 newsegs)
215 /* the context change must make it to memory before the flush, 226 /* the context change must make it to memory before the flush,
216 * so that further SLB misses do the right thing. */ 227 * so that further SLB misses do the right thing. */
217 mb(); 228 mb();
218 on_each_cpu(flush_segments, (void *)(unsigned long)newsegs, 0, 1); 229 on_each_cpu(flush_low_segments, (void *)(unsigned long)newareas, 0, 1);
230
231 return 0;
232}
233
234static int open_high_hpage_areas(struct mm_struct *mm, u16 newareas)
235{
236 unsigned long i;
237
238 BUILD_BUG_ON((sizeof(newareas)*8) != NUM_HIGH_AREAS);
239 BUILD_BUG_ON((sizeof(mm->context.high_htlb_areas)*8)
240 != NUM_HIGH_AREAS);
241
242 newareas &= ~(mm->context.high_htlb_areas);
243 if (! newareas)
244 return 0; /* The areas we want are already open */
245
246 for (i = 0; i < NUM_HIGH_AREAS; i++)
247 if ((1 << i) & newareas)
248 if (prepare_high_area_for_htlb(mm, i) != 0)
249 return -EBUSY;
250
251 mm->context.high_htlb_areas |= newareas;
252
253 /* update the paca copy of the context struct */
254 get_paca()->context = mm->context;
255
256 /* the context change must make it to memory before the flush,
257 * so that further SLB misses do the right thing. */
258 mb();
259 on_each_cpu(flush_high_segments, (void *)(unsigned long)newareas, 0, 1);
219 260
220 return 0; 261 return 0;
221} 262}
222 263
223int prepare_hugepage_range(unsigned long addr, unsigned long len) 264int prepare_hugepage_range(unsigned long addr, unsigned long len)
224{ 265{
225 if (within_hugepage_high_range(addr, len)) 266 int err;
226 return 0; 267
227 else if ((addr < 0x100000000UL) && ((addr+len) < 0x100000000UL)) { 268 if ( (addr+len) < addr )
228 int err; 269 return -EINVAL;
229 /* Yes, we need both tests, in case addr+len overflows 270
230 * 64-bit arithmetic */ 271 if ((addr + len) < 0x100000000UL)
231 err = open_low_hpage_segs(current->mm, 272 err = open_low_hpage_areas(current->mm,
232 LOW_ESID_MASK(addr, len)); 273 LOW_ESID_MASK(addr, len));
233 if (err) 274 else
234 printk(KERN_DEBUG "prepare_hugepage_range(%lx, %lx)" 275 err = open_high_hpage_areas(current->mm,
235 " failed (segs: 0x%04hx)\n", addr, len, 276 HTLB_AREA_MASK(addr, len));
236 LOW_ESID_MASK(addr, len)); 277 if (err) {
278 printk(KERN_DEBUG "prepare_hugepage_range(%lx, %lx)"
279 " failed (lowmask: 0x%04hx, highmask: 0x%04hx)\n",
280 addr, len,
281 LOW_ESID_MASK(addr, len), HTLB_AREA_MASK(addr, len));
237 return err; 282 return err;
238 } 283 }
239 284
240 return -EINVAL; 285 return 0;
241} 286}
242 287
243struct page * 288struct page *
@@ -309,8 +354,8 @@ full_search:
309 vma = find_vma(mm, addr); 354 vma = find_vma(mm, addr);
310 continue; 355 continue;
311 } 356 }
312 if (touches_hugepage_high_range(addr, len)) { 357 if (touches_hugepage_high_range(mm, addr, len)) {
313 addr = TASK_HPAGE_END; 358 addr = ALIGN(addr+1, 1UL<<HTLB_AREA_SHIFT);
314 vma = find_vma(mm, addr); 359 vma = find_vma(mm, addr);
315 continue; 360 continue;
316 } 361 }
@@ -389,8 +434,9 @@ hugepage_recheck:
389 if (touches_hugepage_low_range(mm, addr, len)) { 434 if (touches_hugepage_low_range(mm, addr, len)) {
390 addr = (addr & ((~0) << SID_SHIFT)) - len; 435 addr = (addr & ((~0) << SID_SHIFT)) - len;
391 goto hugepage_recheck; 436 goto hugepage_recheck;
392 } else if (touches_hugepage_high_range(addr, len)) { 437 } else if (touches_hugepage_high_range(mm, addr, len)) {
393 addr = TASK_HPAGE_BASE - len; 438 addr = (addr & ((~0UL) << HTLB_AREA_SHIFT)) - len;
439 goto hugepage_recheck;
394 } 440 }
395 441
396 /* 442 /*
@@ -481,23 +527,28 @@ static unsigned long htlb_get_low_area(unsigned long len, u16 segmask)
481 return -ENOMEM; 527 return -ENOMEM;
482} 528}
483 529
484static unsigned long htlb_get_high_area(unsigned long len) 530static unsigned long htlb_get_high_area(unsigned long len, u16 areamask)
485{ 531{
486 unsigned long addr = TASK_HPAGE_BASE; 532 unsigned long addr = 0x100000000UL;
487 struct vm_area_struct *vma; 533 struct vm_area_struct *vma;
488 534
489 vma = find_vma(current->mm, addr); 535 vma = find_vma(current->mm, addr);
490 for (vma = find_vma(current->mm, addr); 536 while (addr + len <= TASK_SIZE_USER64) {
491 addr + len <= TASK_HPAGE_END;
492 vma = vma->vm_next) {
493 BUG_ON(vma && (addr >= vma->vm_end)); /* invariant */ 537 BUG_ON(vma && (addr >= vma->vm_end)); /* invariant */
494 BUG_ON(! within_hugepage_high_range(addr, len)); 538
539 if (! __within_hugepage_high_range(addr, len, areamask)) {
540 addr = ALIGN(addr+1, 1UL<<HTLB_AREA_SHIFT);
541 vma = find_vma(current->mm, addr);
542 continue;
543 }
495 544
496 if (!vma || (addr + len) <= vma->vm_start) 545 if (!vma || (addr + len) <= vma->vm_start)
497 return addr; 546 return addr;
498 addr = ALIGN(vma->vm_end, HPAGE_SIZE); 547 addr = ALIGN(vma->vm_end, HPAGE_SIZE);
499 /* Because we're in a hugepage region, this alignment 548 /* Depending on segmask this might not be a confirmed
500 * should not skip us over any VMAs */ 549 * hugepage region, so the ALIGN could have skipped
550 * some VMAs */
551 vma = find_vma(current->mm, addr);
501 } 552 }
502 553
503 return -ENOMEM; 554 return -ENOMEM;
@@ -507,6 +558,9 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
507 unsigned long len, unsigned long pgoff, 558 unsigned long len, unsigned long pgoff,
508 unsigned long flags) 559 unsigned long flags)
509{ 560{
561 int lastshift;
562 u16 areamask, curareas;
563
510 if (len & ~HPAGE_MASK) 564 if (len & ~HPAGE_MASK)
511 return -EINVAL; 565 return -EINVAL;
512 566
@@ -514,67 +568,49 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
514 return -EINVAL; 568 return -EINVAL;
515 569
516 if (test_thread_flag(TIF_32BIT)) { 570 if (test_thread_flag(TIF_32BIT)) {
517 int lastshift = 0; 571 curareas = current->mm->context.low_htlb_areas;
518 u16 segmask, cursegs = current->mm->context.htlb_segs;
519 572
520 /* First see if we can do the mapping in the existing 573 /* First see if we can do the mapping in the existing
521 * low hpage segments */ 574 * low areas */
522 addr = htlb_get_low_area(len, cursegs); 575 addr = htlb_get_low_area(len, curareas);
523 if (addr != -ENOMEM) 576 if (addr != -ENOMEM)
524 return addr; 577 return addr;
525 578
526 for (segmask = LOW_ESID_MASK(0x100000000UL-len, len); 579 lastshift = 0;
527 ! lastshift; segmask >>=1) { 580 for (areamask = LOW_ESID_MASK(0x100000000UL-len, len);
528 if (segmask & 1) 581 ! lastshift; areamask >>=1) {
582 if (areamask & 1)
529 lastshift = 1; 583 lastshift = 1;
530 584
531 addr = htlb_get_low_area(len, cursegs | segmask); 585 addr = htlb_get_low_area(len, curareas | areamask);
532 if ((addr != -ENOMEM) 586 if ((addr != -ENOMEM)
533 && open_low_hpage_segs(current->mm, segmask) == 0) 587 && open_low_hpage_areas(current->mm, areamask) == 0)
534 return addr; 588 return addr;
535 } 589 }
536 printk(KERN_DEBUG "hugetlb_get_unmapped_area() unable to open"
537 " enough segments\n");
538 return -ENOMEM;
539 } else { 590 } else {
540 return htlb_get_high_area(len); 591 curareas = current->mm->context.high_htlb_areas;
541 }
542}
543
544void hugetlb_mm_free_pgd(struct mm_struct *mm)
545{
546 int i;
547 pgd_t *pgdir;
548
549 spin_lock(&mm->page_table_lock);
550
551 pgdir = mm->context.huge_pgdir;
552 if (! pgdir)
553 goto out;
554
555 mm->context.huge_pgdir = NULL;
556 592
557 /* cleanup any hugepte pages leftover */ 593 /* First see if we can do the mapping in the existing
558 for (i = 0; i < PTRS_PER_HUGEPGD; i++) { 594 * high areas */
559 pud_t *pud = (pud_t *)(pgdir + i); 595 addr = htlb_get_high_area(len, curareas);
560 596 if (addr != -ENOMEM)
561 if (! pud_none(*pud)) { 597 return addr;
562 pte_t *pte = (pte_t *)pud_page(*pud);
563 struct page *ptepage = virt_to_page(pte);
564 598
565 ptepage->mapping = NULL; 599 lastshift = 0;
600 for (areamask = HTLB_AREA_MASK(TASK_SIZE_USER64-len, len);
601 ! lastshift; areamask >>=1) {
602 if (areamask & 1)
603 lastshift = 1;
566 604
567 BUG_ON(memcmp(pte, empty_zero_page, PAGE_SIZE)); 605 addr = htlb_get_high_area(len, curareas | areamask);
568 kmem_cache_free(zero_cache, pte); 606 if ((addr != -ENOMEM)
607 && open_high_hpage_areas(current->mm, areamask) == 0)
608 return addr;
569 } 609 }
570 pud_clear(pud);
571 } 610 }
572 611 printk(KERN_DEBUG "hugetlb_get_unmapped_area() unable to open"
573 BUG_ON(memcmp(pgdir, empty_zero_page, PAGE_SIZE)); 612 " enough areas\n");
574 kmem_cache_free(zero_cache, pgdir); 613 return -ENOMEM;
575
576 out:
577 spin_unlock(&mm->page_table_lock);
578} 614}
579 615
580int hash_huge_page(struct mm_struct *mm, unsigned long access, 616int hash_huge_page(struct mm_struct *mm, unsigned long access,
diff --git a/arch/ppc64/mm/imalloc.c b/arch/ppc64/mm/imalloc.c
index b6e75b891ac0..c65b87b92756 100644
--- a/arch/ppc64/mm/imalloc.c
+++ b/arch/ppc64/mm/imalloc.c
@@ -31,7 +31,7 @@ static int get_free_im_addr(unsigned long size, unsigned long *im_addr)
31 break; 31 break;
32 if ((unsigned long)tmp->addr >= ioremap_bot) 32 if ((unsigned long)tmp->addr >= ioremap_bot)
33 addr = tmp->size + (unsigned long) tmp->addr; 33 addr = tmp->size + (unsigned long) tmp->addr;
34 if (addr > IMALLOC_END-size) 34 if (addr >= IMALLOC_END-size)
35 return 1; 35 return 1;
36 } 36 }
37 *im_addr = addr; 37 *im_addr = addr;
diff --git a/arch/ppc64/mm/init.c b/arch/ppc64/mm/init.c
index e58a24d42879..c02dc9809ca5 100644
--- a/arch/ppc64/mm/init.c
+++ b/arch/ppc64/mm/init.c
@@ -42,7 +42,6 @@
42 42
43#include <asm/pgalloc.h> 43#include <asm/pgalloc.h>
44#include <asm/page.h> 44#include <asm/page.h>
45#include <asm/abs_addr.h>
46#include <asm/prom.h> 45#include <asm/prom.h>
47#include <asm/lmb.h> 46#include <asm/lmb.h>
48#include <asm/rtas.h> 47#include <asm/rtas.h>
@@ -66,6 +65,14 @@
66#include <asm/vdso.h> 65#include <asm/vdso.h>
67#include <asm/imalloc.h> 66#include <asm/imalloc.h>
68 67
68#if PGTABLE_RANGE > USER_VSID_RANGE
69#warning Limited user VSID range means pagetable space is wasted
70#endif
71
72#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)
73#warning TASK_SIZE is smaller than it needs to be.
74#endif
75
69int mem_init_done; 76int mem_init_done;
70unsigned long ioremap_bot = IMALLOC_BASE; 77unsigned long ioremap_bot = IMALLOC_BASE;
71static unsigned long phbs_io_bot = PHBS_IO_BASE; 78static unsigned long phbs_io_bot = PHBS_IO_BASE;
@@ -159,7 +166,6 @@ static int map_io_page(unsigned long ea, unsigned long pa, int flags)
159 ptep = pte_alloc_kernel(&init_mm, pmdp, ea); 166 ptep = pte_alloc_kernel(&init_mm, pmdp, ea);
160 if (!ptep) 167 if (!ptep)
161 return -ENOMEM; 168 return -ENOMEM;
162 pa = abs_to_phys(pa);
163 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, 169 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT,
164 __pgprot(flags))); 170 __pgprot(flags)));
165 spin_unlock(&init_mm.page_table_lock); 171 spin_unlock(&init_mm.page_table_lock);
@@ -226,7 +232,7 @@ void __iomem * __ioremap(unsigned long addr, unsigned long size,
226 * Before that, we map using addresses going 232 * Before that, we map using addresses going
227 * up from ioremap_bot. imalloc will use 233 * up from ioremap_bot. imalloc will use
228 * the addresses from ioremap_bot through 234 * the addresses from ioremap_bot through
229 * IMALLOC_END (0xE000001fffffffff) 235 * IMALLOC_END
230 * 236 *
231 */ 237 */
232 pa = addr & PAGE_MASK; 238 pa = addr & PAGE_MASK;
@@ -417,12 +423,6 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
417 int index; 423 int index;
418 int err; 424 int err;
419 425
420#ifdef CONFIG_HUGETLB_PAGE
421 /* We leave htlb_segs as it was, but for a fork, we need to
422 * clear the huge_pgdir. */
423 mm->context.huge_pgdir = NULL;
424#endif
425
426again: 426again:
427 if (!idr_pre_get(&mmu_context_idr, GFP_KERNEL)) 427 if (!idr_pre_get(&mmu_context_idr, GFP_KERNEL))
428 return -ENOMEM; 428 return -ENOMEM;
@@ -453,8 +453,6 @@ void destroy_context(struct mm_struct *mm)
453 spin_unlock(&mmu_context_lock); 453 spin_unlock(&mmu_context_lock);
454 454
455 mm->context.id = NO_CONTEXT; 455 mm->context.id = NO_CONTEXT;
456
457 hugetlb_mm_free_pgd(mm);
458} 456}
459 457
460/* 458/*
@@ -484,9 +482,9 @@ void __init mm_init_ppc64(void)
484 for (i = 1; i < lmb.memory.cnt; i++) { 482 for (i = 1; i < lmb.memory.cnt; i++) {
485 unsigned long base, prevbase, prevsize; 483 unsigned long base, prevbase, prevsize;
486 484
487 prevbase = lmb.memory.region[i-1].physbase; 485 prevbase = lmb.memory.region[i-1].base;
488 prevsize = lmb.memory.region[i-1].size; 486 prevsize = lmb.memory.region[i-1].size;
489 base = lmb.memory.region[i].physbase; 487 base = lmb.memory.region[i].base;
490 if (base > (prevbase + prevsize)) { 488 if (base > (prevbase + prevsize)) {
491 io_hole_start = prevbase + prevsize; 489 io_hole_start = prevbase + prevsize;
492 io_hole_size = base - (prevbase + prevsize); 490 io_hole_size = base - (prevbase + prevsize);
@@ -513,11 +511,8 @@ int page_is_ram(unsigned long pfn)
513 for (i=0; i < lmb.memory.cnt; i++) { 511 for (i=0; i < lmb.memory.cnt; i++) {
514 unsigned long base; 512 unsigned long base;
515 513
516#ifdef CONFIG_MSCHUNKS
517 base = lmb.memory.region[i].physbase;
518#else
519 base = lmb.memory.region[i].base; 514 base = lmb.memory.region[i].base;
520#endif 515
521 if ((paddr >= base) && 516 if ((paddr >= base) &&
522 (paddr < (base + lmb.memory.region[i].size))) { 517 (paddr < (base + lmb.memory.region[i].size))) {
523 return 1; 518 return 1;
@@ -547,7 +542,7 @@ void __init do_init_bootmem(void)
547 */ 542 */
548 bootmap_pages = bootmem_bootmap_pages(total_pages); 543 bootmap_pages = bootmem_bootmap_pages(total_pages);
549 544
550 start = abs_to_phys(lmb_alloc(bootmap_pages<<PAGE_SHIFT, PAGE_SIZE)); 545 start = lmb_alloc(bootmap_pages<<PAGE_SHIFT, PAGE_SIZE);
551 BUG_ON(!start); 546 BUG_ON(!start);
552 547
553 boot_mapsize = init_bootmem(start >> PAGE_SHIFT, total_pages); 548 boot_mapsize = init_bootmem(start >> PAGE_SHIFT, total_pages);
@@ -558,25 +553,25 @@ void __init do_init_bootmem(void)
558 * present. 553 * present.
559 */ 554 */
560 for (i=0; i < lmb.memory.cnt; i++) { 555 for (i=0; i < lmb.memory.cnt; i++) {
561 unsigned long physbase, size; 556 unsigned long base, size;
562 unsigned long start_pfn, end_pfn; 557 unsigned long start_pfn, end_pfn;
563 558
564 physbase = lmb.memory.region[i].physbase; 559 base = lmb.memory.region[i].base;
565 size = lmb.memory.region[i].size; 560 size = lmb.memory.region[i].size;
566 561
567 start_pfn = physbase >> PAGE_SHIFT; 562 start_pfn = base >> PAGE_SHIFT;
568 end_pfn = start_pfn + (size >> PAGE_SHIFT); 563 end_pfn = start_pfn + (size >> PAGE_SHIFT);
569 memory_present(0, start_pfn, end_pfn); 564 memory_present(0, start_pfn, end_pfn);
570 565
571 free_bootmem(physbase, size); 566 free_bootmem(base, size);
572 } 567 }
573 568
574 /* reserve the sections we're already using */ 569 /* reserve the sections we're already using */
575 for (i=0; i < lmb.reserved.cnt; i++) { 570 for (i=0; i < lmb.reserved.cnt; i++) {
576 unsigned long physbase = lmb.reserved.region[i].physbase; 571 unsigned long base = lmb.reserved.region[i].base;
577 unsigned long size = lmb.reserved.region[i].size; 572 unsigned long size = lmb.reserved.region[i].size;
578 573
579 reserve_bootmem(physbase, size); 574 reserve_bootmem(base, size);
580 } 575 }
581} 576}
582 577
@@ -615,10 +610,10 @@ static int __init setup_kcore(void)
615 int i; 610 int i;
616 611
617 for (i=0; i < lmb.memory.cnt; i++) { 612 for (i=0; i < lmb.memory.cnt; i++) {
618 unsigned long physbase, size; 613 unsigned long base, size;
619 struct kcore_list *kcore_mem; 614 struct kcore_list *kcore_mem;
620 615
621 physbase = lmb.memory.region[i].physbase; 616 base = lmb.memory.region[i].base;
622 size = lmb.memory.region[i].size; 617 size = lmb.memory.region[i].size;
623 618
624 /* GFP_ATOMIC to avoid might_sleep warnings during boot */ 619 /* GFP_ATOMIC to avoid might_sleep warnings during boot */
@@ -626,7 +621,7 @@ static int __init setup_kcore(void)
626 if (!kcore_mem) 621 if (!kcore_mem)
627 panic("mem_init: kmalloc failed\n"); 622 panic("mem_init: kmalloc failed\n");
628 623
629 kclist_add(kcore_mem, __va(physbase), size); 624 kclist_add(kcore_mem, __va(base), size);
630 } 625 }
631 626
632 kclist_add(&kcore_vmem, (void *)VMALLOC_START, VMALLOC_END-VMALLOC_START); 627 kclist_add(&kcore_vmem, (void *)VMALLOC_START, VMALLOC_END-VMALLOC_START);
@@ -686,9 +681,6 @@ void __init mem_init(void)
686 681
687 mem_init_done = 1; 682 mem_init_done = 1;
688 683
689#ifdef CONFIG_PPC_ISERIES
690 iommu_vio_init();
691#endif
692 /* Initialize the vDSO */ 684 /* Initialize the vDSO */
693 vdso_init(); 685 vdso_init();
694} 686}
@@ -833,23 +825,43 @@ void __iomem * reserve_phb_iospace(unsigned long size)
833 return virt_addr; 825 return virt_addr;
834} 826}
835 827
836kmem_cache_t *zero_cache; 828static void zero_ctor(void *addr, kmem_cache_t *cache, unsigned long flags)
837
838static void zero_ctor(void *pte, kmem_cache_t *cache, unsigned long flags)
839{ 829{
840 memset(pte, 0, PAGE_SIZE); 830 memset(addr, 0, kmem_cache_size(cache));
841} 831}
842 832
833static const int pgtable_cache_size[2] = {
834 PTE_TABLE_SIZE, PMD_TABLE_SIZE
835};
836static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = {
837 "pgd_pte_cache", "pud_pmd_cache",
838};
839
840kmem_cache_t *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)];
841
843void pgtable_cache_init(void) 842void pgtable_cache_init(void)
844{ 843{
845 zero_cache = kmem_cache_create("zero", 844 int i;
846 PAGE_SIZE, 845
847 0, 846 BUILD_BUG_ON(PTE_TABLE_SIZE != pgtable_cache_size[PTE_CACHE_NUM]);
848 SLAB_HWCACHE_ALIGN | SLAB_MUST_HWCACHE_ALIGN, 847 BUILD_BUG_ON(PMD_TABLE_SIZE != pgtable_cache_size[PMD_CACHE_NUM]);
849 zero_ctor, 848 BUILD_BUG_ON(PUD_TABLE_SIZE != pgtable_cache_size[PUD_CACHE_NUM]);
850 NULL); 849 BUILD_BUG_ON(PGD_TABLE_SIZE != pgtable_cache_size[PGD_CACHE_NUM]);
851 if (!zero_cache) 850
852 panic("pgtable_cache_init(): could not create zero_cache!\n"); 851 for (i = 0; i < ARRAY_SIZE(pgtable_cache_size); i++) {
852 int size = pgtable_cache_size[i];
853 const char *name = pgtable_cache_name[i];
854
855 pgtable_cache[i] = kmem_cache_create(name,
856 size, size,
857 SLAB_HWCACHE_ALIGN
858 | SLAB_MUST_HWCACHE_ALIGN,
859 zero_ctor,
860 NULL);
861 if (! pgtable_cache[i])
862 panic("pgtable_cache_init(): could not create %s!\n",
863 name);
864 }
853} 865}
854 866
855pgprot_t phys_mem_access_prot(struct file *file, unsigned long addr, 867pgprot_t phys_mem_access_prot(struct file *file, unsigned long addr,
diff --git a/arch/ppc64/mm/numa.c b/arch/ppc64/mm/numa.c
index 0b191f2de016..c3116f0d788c 100644
--- a/arch/ppc64/mm/numa.c
+++ b/arch/ppc64/mm/numa.c
@@ -671,7 +671,7 @@ new_range:
671 * Mark reserved regions on this node 671 * Mark reserved regions on this node
672 */ 672 */
673 for (i = 0; i < lmb.reserved.cnt; i++) { 673 for (i = 0; i < lmb.reserved.cnt; i++) {
674 unsigned long physbase = lmb.reserved.region[i].physbase; 674 unsigned long physbase = lmb.reserved.region[i].base;
675 unsigned long size = lmb.reserved.region[i].size; 675 unsigned long size = lmb.reserved.region[i].size;
676 676
677 if (pa_to_nid(physbase) != nid && 677 if (pa_to_nid(physbase) != nid &&
diff --git a/arch/ppc64/mm/slb_low.S b/arch/ppc64/mm/slb_low.S
index 8379d678f70f..698d6b9ed6d1 100644
--- a/arch/ppc64/mm/slb_low.S
+++ b/arch/ppc64/mm/slb_low.S
@@ -89,32 +89,29 @@ END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE)
89 b 9f 89 b 9f
90 90
910: /* user address: proto-VSID = context<<15 | ESID */ 910: /* user address: proto-VSID = context<<15 | ESID */
92 li r11,SLB_VSID_USER 92 srdi. r9,r3,USER_ESID_BITS
93
94 srdi. r9,r3,13
95 bne- 8f /* invalid ea bits set */ 93 bne- 8f /* invalid ea bits set */
96 94
97#ifdef CONFIG_HUGETLB_PAGE 95#ifdef CONFIG_HUGETLB_PAGE
98BEGIN_FTR_SECTION 96BEGIN_FTR_SECTION
99 /* check against the hugepage ranges */ 97 lhz r9,PACAHIGHHTLBAREAS(r13)
100 cmpldi r3,(TASK_HPAGE_END>>SID_SHIFT) 98 srdi r11,r3,(HTLB_AREA_SHIFT-SID_SHIFT)
101 bge 6f /* >= TASK_HPAGE_END */ 99 srd r9,r9,r11
102 cmpldi r3,(TASK_HPAGE_BASE>>SID_SHIFT) 100 lhz r11,PACALOWHTLBAREAS(r13)
103 bge 5f /* TASK_HPAGE_BASE..TASK_HPAGE_END */ 101 srd r11,r11,r3
104 cmpldi r3,16 102 or r9,r9,r11
105 bge 6f /* 4GB..TASK_HPAGE_BASE */ 103END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE)
106 104#endif /* CONFIG_HUGETLB_PAGE */
107 lhz r9,PACAHTLBSEGS(r13) 105
108 srd r9,r9,r3 106 li r11,SLB_VSID_USER
109 andi. r9,r9,1 107
110 beq 6f 108#ifdef CONFIG_HUGETLB_PAGE
111 109BEGIN_FTR_SECTION
1125: /* this is a hugepage user address */ 110 rldimi r11,r9,8,55 /* shift masked bit into SLB_VSID_L */
113 li r11,(SLB_VSID_USER|SLB_VSID_L)
114END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE) 111END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE)
115#endif /* CONFIG_HUGETLB_PAGE */ 112#endif /* CONFIG_HUGETLB_PAGE */
116 113
1176: ld r9,PACACONTEXTID(r13) 114 ld r9,PACACONTEXTID(r13)
118 rldimi r3,r9,USER_ESID_BITS,0 115 rldimi r3,r9,USER_ESID_BITS,0
119 116
1209: /* r3 = protovsid, r11 = flags, r10 = esid_data, cr7 = <>KERNELBASE */ 1179: /* r3 = protovsid, r11 = flags, r10 = esid_data, cr7 = <>KERNELBASE */
diff --git a/arch/ppc64/mm/tlb.c b/arch/ppc64/mm/tlb.c
index 26f0172c4527..d8a6593a13f0 100644
--- a/arch/ppc64/mm/tlb.c
+++ b/arch/ppc64/mm/tlb.c
@@ -41,7 +41,58 @@ DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
41DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur); 41DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
42unsigned long pte_freelist_forced_free; 42unsigned long pte_freelist_forced_free;
43 43
44void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage) 44struct pte_freelist_batch
45{
46 struct rcu_head rcu;
47 unsigned int index;
48 pgtable_free_t tables[0];
49};
50
51DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
52unsigned long pte_freelist_forced_free;
53
54#define PTE_FREELIST_SIZE \
55 ((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \
56 / sizeof(pgtable_free_t))
57
58#ifdef CONFIG_SMP
59static void pte_free_smp_sync(void *arg)
60{
61 /* Do nothing, just ensure we sync with all CPUs */
62}
63#endif
64
65/* This is only called when we are critically out of memory
66 * (and fail to get a page in pte_free_tlb).
67 */
68static void pgtable_free_now(pgtable_free_t pgf)
69{
70 pte_freelist_forced_free++;
71
72 smp_call_function(pte_free_smp_sync, NULL, 0, 1);
73
74 pgtable_free(pgf);
75}
76
77static void pte_free_rcu_callback(struct rcu_head *head)
78{
79 struct pte_freelist_batch *batch =
80 container_of(head, struct pte_freelist_batch, rcu);
81 unsigned int i;
82
83 for (i = 0; i < batch->index; i++)
84 pgtable_free(batch->tables[i]);
85
86 free_page((unsigned long)batch);
87}
88
89static void pte_free_submit(struct pte_freelist_batch *batch)
90{
91 INIT_RCU_HEAD(&batch->rcu);
92 call_rcu(&batch->rcu, pte_free_rcu_callback);
93}
94
95void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf)
45{ 96{
46 /* This is safe as we are holding page_table_lock */ 97 /* This is safe as we are holding page_table_lock */
47 cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id()); 98 cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id());
@@ -49,19 +100,19 @@ void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage)
49 100
50 if (atomic_read(&tlb->mm->mm_users) < 2 || 101 if (atomic_read(&tlb->mm->mm_users) < 2 ||
51 cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) { 102 cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) {
52 pte_free(ptepage); 103 pgtable_free(pgf);
53 return; 104 return;
54 } 105 }
55 106
56 if (*batchp == NULL) { 107 if (*batchp == NULL) {
57 *batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC); 108 *batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC);
58 if (*batchp == NULL) { 109 if (*batchp == NULL) {
59 pte_free_now(ptepage); 110 pgtable_free_now(pgf);
60 return; 111 return;
61 } 112 }
62 (*batchp)->index = 0; 113 (*batchp)->index = 0;
63 } 114 }
64 (*batchp)->pages[(*batchp)->index++] = ptepage; 115 (*batchp)->tables[(*batchp)->index++] = pgf;
65 if ((*batchp)->index == PTE_FREELIST_SIZE) { 116 if ((*batchp)->index == PTE_FREELIST_SIZE) {
66 pte_free_submit(*batchp); 117 pte_free_submit(*batchp);
67 *batchp = NULL; 118 *batchp = NULL;
@@ -132,42 +183,6 @@ void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
132 put_cpu(); 183 put_cpu();
133} 184}
134 185
135#ifdef CONFIG_SMP
136static void pte_free_smp_sync(void *arg)
137{
138 /* Do nothing, just ensure we sync with all CPUs */
139}
140#endif
141
142/* This is only called when we are critically out of memory
143 * (and fail to get a page in pte_free_tlb).
144 */
145void pte_free_now(struct page *ptepage)
146{
147 pte_freelist_forced_free++;
148
149 smp_call_function(pte_free_smp_sync, NULL, 0, 1);
150
151 pte_free(ptepage);
152}
153
154static void pte_free_rcu_callback(struct rcu_head *head)
155{
156 struct pte_freelist_batch *batch =
157 container_of(head, struct pte_freelist_batch, rcu);
158 unsigned int i;
159
160 for (i = 0; i < batch->index; i++)
161 pte_free(batch->pages[i]);
162 free_page((unsigned long)batch);
163}
164
165void pte_free_submit(struct pte_freelist_batch *batch)
166{
167 INIT_RCU_HEAD(&batch->rcu);
168 call_rcu(&batch->rcu, pte_free_rcu_callback);
169}
170
171void pte_free_finish(void) 186void pte_free_finish(void)
172{ 187{
173 /* This is safe as we are holding page_table_lock */ 188 /* This is safe as we are holding page_table_lock */
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-05 13:27:42 -0400