aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm
diff options
context:
space:
mode:
authorMichel Lespinasse <walken@google.com>2013-04-29 14:53:52 -0400
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>2013-04-29 21:05:10 -0400
commit34d07177b802e963f3b14bf3dd8caf294f41fea7 (patch)
tree38a20210326cde47f836875e71f59b63577a02ad /arch/powerpc/mm
parent3925f46bb5902ba9f227591584e27acb6a32c9b0 (diff)
mm: remove free_area_cache use in powerpc architecture
As all other architectures have been converted to use vm_unmapped_area(), we are about to retire the free_area_cache. This change simply removes the use of that cache in slice_get_unmapped_area(), which will most certainly have a performance cost. Next one will convert that function to use the vm_unmapped_area() infrastructure and regain the performance. Signed-off-by: Michel Lespinasse <walken@google.com> Acked-by: Rik van Riel <riel@redhat.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Paul Mackerras <paulus@samba.org> Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Diffstat (limited to 'arch/powerpc/mm')
-rw-r--r--arch/powerpc/mm/hugetlbpage.c2
-rw-r--r--arch/powerpc/mm/slice.c108
2 files changed, 20 insertions, 90 deletions
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 1a6de0a7d8eb..5dc52d803ed8 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -742,7 +742,7 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
742 struct hstate *hstate = hstate_file(file); 742 struct hstate *hstate = hstate_file(file);
743 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); 743 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
744 744
745 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0); 745 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
746} 746}
747#endif 747#endif
748 748
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index cf9dada734b6..999a74f25ebe 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -240,23 +240,15 @@ static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psiz
240static unsigned long slice_find_area_bottomup(struct mm_struct *mm, 240static unsigned long slice_find_area_bottomup(struct mm_struct *mm,
241 unsigned long len, 241 unsigned long len,
242 struct slice_mask available, 242 struct slice_mask available,
243 int psize, int use_cache) 243 int psize)
244{ 244{
245 struct vm_area_struct *vma; 245 struct vm_area_struct *vma;
246 unsigned long start_addr, addr; 246 unsigned long addr;
247 struct slice_mask mask; 247 struct slice_mask mask;
248 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); 248 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
249 249
250 if (use_cache) { 250 addr = TASK_UNMAPPED_BASE;
251 if (len <= mm->cached_hole_size) {
252 start_addr = addr = TASK_UNMAPPED_BASE;
253 mm->cached_hole_size = 0;
254 } else
255 start_addr = addr = mm->free_area_cache;
256 } else
257 start_addr = addr = TASK_UNMAPPED_BASE;
258 251
259full_search:
260 for (;;) { 252 for (;;) {
261 addr = _ALIGN_UP(addr, 1ul << pshift); 253 addr = _ALIGN_UP(addr, 1ul << pshift);
262 if ((TASK_SIZE - len) < addr) 254 if ((TASK_SIZE - len) < addr)
@@ -272,63 +264,24 @@ full_search:
272 addr = _ALIGN_UP(addr + 1, 1ul << SLICE_HIGH_SHIFT); 264 addr = _ALIGN_UP(addr + 1, 1ul << SLICE_HIGH_SHIFT);
273 continue; 265 continue;
274 } 266 }
275 if (!vma || addr + len <= vma->vm_start) { 267 if (!vma || addr + len <= vma->vm_start)
276 /*
277 * Remember the place where we stopped the search:
278 */
279 if (use_cache)
280 mm->free_area_cache = addr + len;
281 return addr; 268 return addr;
282 }
283 if (use_cache && (addr + mm->cached_hole_size) < vma->vm_start)
284 mm->cached_hole_size = vma->vm_start - addr;
285 addr = vma->vm_end; 269 addr = vma->vm_end;
286 } 270 }
287 271
288 /* Make sure we didn't miss any holes */
289 if (use_cache && start_addr != TASK_UNMAPPED_BASE) {
290 start_addr = addr = TASK_UNMAPPED_BASE;
291 mm->cached_hole_size = 0;
292 goto full_search;
293 }
294 return -ENOMEM; 272 return -ENOMEM;
295} 273}
296 274
297static unsigned long slice_find_area_topdown(struct mm_struct *mm, 275static unsigned long slice_find_area_topdown(struct mm_struct *mm,
298 unsigned long len, 276 unsigned long len,
299 struct slice_mask available, 277 struct slice_mask available,
300 int psize, int use_cache) 278 int psize)
301{ 279{
302 struct vm_area_struct *vma; 280 struct vm_area_struct *vma;
303 unsigned long addr; 281 unsigned long addr;
304 struct slice_mask mask; 282 struct slice_mask mask;
305 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); 283 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
306 284
307 /* check if free_area_cache is useful for us */
308 if (use_cache) {
309 if (len <= mm->cached_hole_size) {
310 mm->cached_hole_size = 0;
311 mm->free_area_cache = mm->mmap_base;
312 }
313
314 /* either no address requested or can't fit in requested
315 * address hole
316 */
317 addr = mm->free_area_cache;
318
319 /* make sure it can fit in the remaining address space */
320 if (addr > len) {
321 addr = _ALIGN_DOWN(addr - len, 1ul << pshift);
322 mask = slice_range_to_mask(addr, len);
323 if (slice_check_fit(mask, available) &&
324 slice_area_is_free(mm, addr, len))
325 /* remember the address as a hint for
326 * next time
327 */
328 return (mm->free_area_cache = addr);
329 }
330 }
331
332 addr = mm->mmap_base; 285 addr = mm->mmap_base;
333 while (addr > len) { 286 while (addr > len) {
334 /* Go down by chunk size */ 287 /* Go down by chunk size */
@@ -352,16 +305,8 @@ static unsigned long slice_find_area_topdown(struct mm_struct *mm,
352 * return with success: 305 * return with success:
353 */ 306 */
354 vma = find_vma(mm, addr); 307 vma = find_vma(mm, addr);
355 if (!vma || (addr + len) <= vma->vm_start) { 308 if (!vma || (addr + len) <= vma->vm_start)
356 /* remember the address as a hint for next time */
357 if (use_cache)
358 mm->free_area_cache = addr;
359 return addr; 309 return addr;
360 }
361
362 /* remember the largest hole we saw so far */
363 if (use_cache && (addr + mm->cached_hole_size) < vma->vm_start)
364 mm->cached_hole_size = vma->vm_start - addr;
365 310
366 /* try just below the current vma->vm_start */ 311 /* try just below the current vma->vm_start */
367 addr = vma->vm_start; 312 addr = vma->vm_start;
@@ -373,28 +318,18 @@ static unsigned long slice_find_area_topdown(struct mm_struct *mm,
373 * can happen with large stack limits and large mmap() 318 * can happen with large stack limits and large mmap()
374 * allocations. 319 * allocations.
375 */ 320 */
376 addr = slice_find_area_bottomup(mm, len, available, psize, 0); 321 return slice_find_area_bottomup(mm, len, available, psize);
377
378 /*
379 * Restore the topdown base:
380 */
381 if (use_cache) {
382 mm->free_area_cache = mm->mmap_base;
383 mm->cached_hole_size = ~0UL;
384 }
385
386 return addr;
387} 322}
388 323
389 324
390static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len, 325static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
391 struct slice_mask mask, int psize, 326 struct slice_mask mask, int psize,
392 int topdown, int use_cache) 327 int topdown)
393{ 328{
394 if (topdown) 329 if (topdown)
395 return slice_find_area_topdown(mm, len, mask, psize, use_cache); 330 return slice_find_area_topdown(mm, len, mask, psize);
396 else 331 else
397 return slice_find_area_bottomup(mm, len, mask, psize, use_cache); 332 return slice_find_area_bottomup(mm, len, mask, psize);
398} 333}
399 334
400#define or_mask(dst, src) do { \ 335#define or_mask(dst, src) do { \
@@ -415,7 +350,7 @@ static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
415 350
416unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, 351unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
417 unsigned long flags, unsigned int psize, 352 unsigned long flags, unsigned int psize,
418 int topdown, int use_cache) 353 int topdown)
419{ 354{
420 struct slice_mask mask = {0, 0}; 355 struct slice_mask mask = {0, 0};
421 struct slice_mask good_mask; 356 struct slice_mask good_mask;
@@ -430,8 +365,8 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
430 BUG_ON(mm->task_size == 0); 365 BUG_ON(mm->task_size == 0);
431 366
432 slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize); 367 slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize);
433 slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d, use_cache=%d\n", 368 slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d\n",
434 addr, len, flags, topdown, use_cache); 369 addr, len, flags, topdown);
435 370
436 if (len > mm->task_size) 371 if (len > mm->task_size)
437 return -ENOMEM; 372 return -ENOMEM;
@@ -503,8 +438,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
503 /* Now let's see if we can find something in the existing 438 /* Now let's see if we can find something in the existing
504 * slices for that size 439 * slices for that size
505 */ 440 */
506 newaddr = slice_find_area(mm, len, good_mask, psize, topdown, 441 newaddr = slice_find_area(mm, len, good_mask, psize, topdown);
507 use_cache);
508 if (newaddr != -ENOMEM) { 442 if (newaddr != -ENOMEM) {
509 /* Found within the good mask, we don't have to setup, 443 /* Found within the good mask, we don't have to setup,
510 * we thus return directly 444 * we thus return directly
@@ -536,8 +470,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
536 * anywhere in the good area. 470 * anywhere in the good area.
537 */ 471 */
538 if (addr) { 472 if (addr) {
539 addr = slice_find_area(mm, len, good_mask, psize, topdown, 473 addr = slice_find_area(mm, len, good_mask, psize, topdown);
540 use_cache);
541 if (addr != -ENOMEM) { 474 if (addr != -ENOMEM) {
542 slice_dbg(" found area at 0x%lx\n", addr); 475 slice_dbg(" found area at 0x%lx\n", addr);
543 return addr; 476 return addr;
@@ -547,15 +480,14 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
547 /* Now let's see if we can find something in the existing slices 480 /* Now let's see if we can find something in the existing slices
548 * for that size plus free slices 481 * for that size plus free slices
549 */ 482 */
550 addr = slice_find_area(mm, len, potential_mask, psize, topdown, 483 addr = slice_find_area(mm, len, potential_mask, psize, topdown);
551 use_cache);
552 484
553#ifdef CONFIG_PPC_64K_PAGES 485#ifdef CONFIG_PPC_64K_PAGES
554 if (addr == -ENOMEM && psize == MMU_PAGE_64K) { 486 if (addr == -ENOMEM && psize == MMU_PAGE_64K) {
555 /* retry the search with 4k-page slices included */ 487 /* retry the search with 4k-page slices included */
556 or_mask(potential_mask, compat_mask); 488 or_mask(potential_mask, compat_mask);
557 addr = slice_find_area(mm, len, potential_mask, psize, 489 addr = slice_find_area(mm, len, potential_mask, psize,
558 topdown, use_cache); 490 topdown);
559 } 491 }
560#endif 492#endif
561 493
@@ -586,8 +518,7 @@ unsigned long arch_get_unmapped_area(struct file *filp,
586 unsigned long flags) 518 unsigned long flags)
587{ 519{
588 return slice_get_unmapped_area(addr, len, flags, 520 return slice_get_unmapped_area(addr, len, flags,
589 current->mm->context.user_psize, 521 current->mm->context.user_psize, 0);
590 0, 1);
591} 522}
592 523
593unsigned long arch_get_unmapped_area_topdown(struct file *filp, 524unsigned long arch_get_unmapped_area_topdown(struct file *filp,
@@ -597,8 +528,7 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp,
597 const unsigned long flags) 528 const unsigned long flags)
598{ 529{
599 return slice_get_unmapped_area(addr0, len, flags, 530 return slice_get_unmapped_area(addr0, len, flags,
600 current->mm->context.user_psize, 531 current->mm->context.user_psize, 1);
601 1, 1);
602} 532}
603 533
604unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr) 534unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr)