diff options
author | Michel Lespinasse <walken@google.com> | 2013-04-29 14:53:52 -0400 |
---|---|---|
committer | Benjamin Herrenschmidt <benh@kernel.crashing.org> | 2013-04-29 21:05:10 -0400 |
commit | 34d07177b802e963f3b14bf3dd8caf294f41fea7 (patch) | |
tree | 38a20210326cde47f836875e71f59b63577a02ad | |
parent | 3925f46bb5902ba9f227591584e27acb6a32c9b0 (diff) |
mm: remove free_area_cache use in powerpc architecture
As all other architectures have been converted to use vm_unmapped_area(),
we are about to retire the free_area_cache.
This change simply removes the use of that cache in
slice_get_unmapped_area(), which will most certainly have a
performance cost. Next one will convert that function to use the
vm_unmapped_area() infrastructure and regain the performance.
Signed-off-by: Michel Lespinasse <walken@google.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
-rw-r--r-- | arch/powerpc/include/asm/page_64.h | 3 | ||||
-rw-r--r-- | arch/powerpc/mm/hugetlbpage.c | 2 | ||||
-rw-r--r-- | arch/powerpc/mm/slice.c | 108 | ||||
-rw-r--r-- | arch/powerpc/platforms/cell/spufs/file.c | 2 |
4 files changed, 22 insertions, 93 deletions
diff --git a/arch/powerpc/include/asm/page_64.h b/arch/powerpc/include/asm/page_64.h index cd915d6b093d..88693cef4f3d 100644 --- a/arch/powerpc/include/asm/page_64.h +++ b/arch/powerpc/include/asm/page_64.h | |||
@@ -99,8 +99,7 @@ extern unsigned long slice_get_unmapped_area(unsigned long addr, | |||
99 | unsigned long len, | 99 | unsigned long len, |
100 | unsigned long flags, | 100 | unsigned long flags, |
101 | unsigned int psize, | 101 | unsigned int psize, |
102 | int topdown, | 102 | int topdown); |
103 | int use_cache); | ||
104 | 103 | ||
105 | extern unsigned int get_slice_psize(struct mm_struct *mm, | 104 | extern unsigned int get_slice_psize(struct mm_struct *mm, |
106 | unsigned long addr); | 105 | unsigned long addr); |
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 1a6de0a7d8eb..5dc52d803ed8 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c | |||
@@ -742,7 +742,7 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, | |||
742 | struct hstate *hstate = hstate_file(file); | 742 | struct hstate *hstate = hstate_file(file); |
743 | int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); | 743 | int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); |
744 | 744 | ||
745 | return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0); | 745 | return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1); |
746 | } | 746 | } |
747 | #endif | 747 | #endif |
748 | 748 | ||
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c index cf9dada734b6..999a74f25ebe 100644 --- a/arch/powerpc/mm/slice.c +++ b/arch/powerpc/mm/slice.c | |||
@@ -240,23 +240,15 @@ static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psiz | |||
240 | static unsigned long slice_find_area_bottomup(struct mm_struct *mm, | 240 | static unsigned long slice_find_area_bottomup(struct mm_struct *mm, |
241 | unsigned long len, | 241 | unsigned long len, |
242 | struct slice_mask available, | 242 | struct slice_mask available, |
243 | int psize, int use_cache) | 243 | int psize) |
244 | { | 244 | { |
245 | struct vm_area_struct *vma; | 245 | struct vm_area_struct *vma; |
246 | unsigned long start_addr, addr; | 246 | unsigned long addr; |
247 | struct slice_mask mask; | 247 | struct slice_mask mask; |
248 | int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); | 248 | int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); |
249 | 249 | ||
250 | if (use_cache) { | 250 | addr = TASK_UNMAPPED_BASE; |
251 | if (len <= mm->cached_hole_size) { | ||
252 | start_addr = addr = TASK_UNMAPPED_BASE; | ||
253 | mm->cached_hole_size = 0; | ||
254 | } else | ||
255 | start_addr = addr = mm->free_area_cache; | ||
256 | } else | ||
257 | start_addr = addr = TASK_UNMAPPED_BASE; | ||
258 | 251 | ||
259 | full_search: | ||
260 | for (;;) { | 252 | for (;;) { |
261 | addr = _ALIGN_UP(addr, 1ul << pshift); | 253 | addr = _ALIGN_UP(addr, 1ul << pshift); |
262 | if ((TASK_SIZE - len) < addr) | 254 | if ((TASK_SIZE - len) < addr) |
@@ -272,63 +264,24 @@ full_search: | |||
272 | addr = _ALIGN_UP(addr + 1, 1ul << SLICE_HIGH_SHIFT); | 264 | addr = _ALIGN_UP(addr + 1, 1ul << SLICE_HIGH_SHIFT); |
273 | continue; | 265 | continue; |
274 | } | 266 | } |
275 | if (!vma || addr + len <= vma->vm_start) { | 267 | if (!vma || addr + len <= vma->vm_start) |
276 | /* | ||
277 | * Remember the place where we stopped the search: | ||
278 | */ | ||
279 | if (use_cache) | ||
280 | mm->free_area_cache = addr + len; | ||
281 | return addr; | 268 | return addr; |
282 | } | ||
283 | if (use_cache && (addr + mm->cached_hole_size) < vma->vm_start) | ||
284 | mm->cached_hole_size = vma->vm_start - addr; | ||
285 | addr = vma->vm_end; | 269 | addr = vma->vm_end; |
286 | } | 270 | } |
287 | 271 | ||
288 | /* Make sure we didn't miss any holes */ | ||
289 | if (use_cache && start_addr != TASK_UNMAPPED_BASE) { | ||
290 | start_addr = addr = TASK_UNMAPPED_BASE; | ||
291 | mm->cached_hole_size = 0; | ||
292 | goto full_search; | ||
293 | } | ||
294 | return -ENOMEM; | 272 | return -ENOMEM; |
295 | } | 273 | } |
296 | 274 | ||
297 | static unsigned long slice_find_area_topdown(struct mm_struct *mm, | 275 | static unsigned long slice_find_area_topdown(struct mm_struct *mm, |
298 | unsigned long len, | 276 | unsigned long len, |
299 | struct slice_mask available, | 277 | struct slice_mask available, |
300 | int psize, int use_cache) | 278 | int psize) |
301 | { | 279 | { |
302 | struct vm_area_struct *vma; | 280 | struct vm_area_struct *vma; |
303 | unsigned long addr; | 281 | unsigned long addr; |
304 | struct slice_mask mask; | 282 | struct slice_mask mask; |
305 | int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); | 283 | int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); |
306 | 284 | ||
307 | /* check if free_area_cache is useful for us */ | ||
308 | if (use_cache) { | ||
309 | if (len <= mm->cached_hole_size) { | ||
310 | mm->cached_hole_size = 0; | ||
311 | mm->free_area_cache = mm->mmap_base; | ||
312 | } | ||
313 | |||
314 | /* either no address requested or can't fit in requested | ||
315 | * address hole | ||
316 | */ | ||
317 | addr = mm->free_area_cache; | ||
318 | |||
319 | /* make sure it can fit in the remaining address space */ | ||
320 | if (addr > len) { | ||
321 | addr = _ALIGN_DOWN(addr - len, 1ul << pshift); | ||
322 | mask = slice_range_to_mask(addr, len); | ||
323 | if (slice_check_fit(mask, available) && | ||
324 | slice_area_is_free(mm, addr, len)) | ||
325 | /* remember the address as a hint for | ||
326 | * next time | ||
327 | */ | ||
328 | return (mm->free_area_cache = addr); | ||
329 | } | ||
330 | } | ||
331 | |||
332 | addr = mm->mmap_base; | 285 | addr = mm->mmap_base; |
333 | while (addr > len) { | 286 | while (addr > len) { |
334 | /* Go down by chunk size */ | 287 | /* Go down by chunk size */ |
@@ -352,16 +305,8 @@ static unsigned long slice_find_area_topdown(struct mm_struct *mm, | |||
352 | * return with success: | 305 | * return with success: |
353 | */ | 306 | */ |
354 | vma = find_vma(mm, addr); | 307 | vma = find_vma(mm, addr); |
355 | if (!vma || (addr + len) <= vma->vm_start) { | 308 | if (!vma || (addr + len) <= vma->vm_start) |
356 | /* remember the address as a hint for next time */ | ||
357 | if (use_cache) | ||
358 | mm->free_area_cache = addr; | ||
359 | return addr; | 309 | return addr; |
360 | } | ||
361 | |||
362 | /* remember the largest hole we saw so far */ | ||
363 | if (use_cache && (addr + mm->cached_hole_size) < vma->vm_start) | ||
364 | mm->cached_hole_size = vma->vm_start - addr; | ||
365 | 310 | ||
366 | /* try just below the current vma->vm_start */ | 311 | /* try just below the current vma->vm_start */ |
367 | addr = vma->vm_start; | 312 | addr = vma->vm_start; |
@@ -373,28 +318,18 @@ static unsigned long slice_find_area_topdown(struct mm_struct *mm, | |||
373 | * can happen with large stack limits and large mmap() | 318 | * can happen with large stack limits and large mmap() |
374 | * allocations. | 319 | * allocations. |
375 | */ | 320 | */ |
376 | addr = slice_find_area_bottomup(mm, len, available, psize, 0); | 321 | return slice_find_area_bottomup(mm, len, available, psize); |
377 | |||
378 | /* | ||
379 | * Restore the topdown base: | ||
380 | */ | ||
381 | if (use_cache) { | ||
382 | mm->free_area_cache = mm->mmap_base; | ||
383 | mm->cached_hole_size = ~0UL; | ||
384 | } | ||
385 | |||
386 | return addr; | ||
387 | } | 322 | } |
388 | 323 | ||
389 | 324 | ||
390 | static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len, | 325 | static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len, |
391 | struct slice_mask mask, int psize, | 326 | struct slice_mask mask, int psize, |
392 | int topdown, int use_cache) | 327 | int topdown) |
393 | { | 328 | { |
394 | if (topdown) | 329 | if (topdown) |
395 | return slice_find_area_topdown(mm, len, mask, psize, use_cache); | 330 | return slice_find_area_topdown(mm, len, mask, psize); |
396 | else | 331 | else |
397 | return slice_find_area_bottomup(mm, len, mask, psize, use_cache); | 332 | return slice_find_area_bottomup(mm, len, mask, psize); |
398 | } | 333 | } |
399 | 334 | ||
400 | #define or_mask(dst, src) do { \ | 335 | #define or_mask(dst, src) do { \ |
@@ -415,7 +350,7 @@ static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len, | |||
415 | 350 | ||
416 | unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, | 351 | unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, |
417 | unsigned long flags, unsigned int psize, | 352 | unsigned long flags, unsigned int psize, |
418 | int topdown, int use_cache) | 353 | int topdown) |
419 | { | 354 | { |
420 | struct slice_mask mask = {0, 0}; | 355 | struct slice_mask mask = {0, 0}; |
421 | struct slice_mask good_mask; | 356 | struct slice_mask good_mask; |
@@ -430,8 +365,8 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, | |||
430 | BUG_ON(mm->task_size == 0); | 365 | BUG_ON(mm->task_size == 0); |
431 | 366 | ||
432 | slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize); | 367 | slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize); |
433 | slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d, use_cache=%d\n", | 368 | slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d\n", |
434 | addr, len, flags, topdown, use_cache); | 369 | addr, len, flags, topdown); |
435 | 370 | ||
436 | if (len > mm->task_size) | 371 | if (len > mm->task_size) |
437 | return -ENOMEM; | 372 | return -ENOMEM; |
@@ -503,8 +438,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, | |||
503 | /* Now let's see if we can find something in the existing | 438 | /* Now let's see if we can find something in the existing |
504 | * slices for that size | 439 | * slices for that size |
505 | */ | 440 | */ |
506 | newaddr = slice_find_area(mm, len, good_mask, psize, topdown, | 441 | newaddr = slice_find_area(mm, len, good_mask, psize, topdown); |
507 | use_cache); | ||
508 | if (newaddr != -ENOMEM) { | 442 | if (newaddr != -ENOMEM) { |
509 | /* Found within the good mask, we don't have to setup, | 443 | /* Found within the good mask, we don't have to setup, |
510 | * we thus return directly | 444 | * we thus return directly |
@@ -536,8 +470,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, | |||
536 | * anywhere in the good area. | 470 | * anywhere in the good area. |
537 | */ | 471 | */ |
538 | if (addr) { | 472 | if (addr) { |
539 | addr = slice_find_area(mm, len, good_mask, psize, topdown, | 473 | addr = slice_find_area(mm, len, good_mask, psize, topdown); |
540 | use_cache); | ||
541 | if (addr != -ENOMEM) { | 474 | if (addr != -ENOMEM) { |
542 | slice_dbg(" found area at 0x%lx\n", addr); | 475 | slice_dbg(" found area at 0x%lx\n", addr); |
543 | return addr; | 476 | return addr; |
@@ -547,15 +480,14 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, | |||
547 | /* Now let's see if we can find something in the existing slices | 480 | /* Now let's see if we can find something in the existing slices |
548 | * for that size plus free slices | 481 | * for that size plus free slices |
549 | */ | 482 | */ |
550 | addr = slice_find_area(mm, len, potential_mask, psize, topdown, | 483 | addr = slice_find_area(mm, len, potential_mask, psize, topdown); |
551 | use_cache); | ||
552 | 484 | ||
553 | #ifdef CONFIG_PPC_64K_PAGES | 485 | #ifdef CONFIG_PPC_64K_PAGES |
554 | if (addr == -ENOMEM && psize == MMU_PAGE_64K) { | 486 | if (addr == -ENOMEM && psize == MMU_PAGE_64K) { |
555 | /* retry the search with 4k-page slices included */ | 487 | /* retry the search with 4k-page slices included */ |
556 | or_mask(potential_mask, compat_mask); | 488 | or_mask(potential_mask, compat_mask); |
557 | addr = slice_find_area(mm, len, potential_mask, psize, | 489 | addr = slice_find_area(mm, len, potential_mask, psize, |
558 | topdown, use_cache); | 490 | topdown); |
559 | } | 491 | } |
560 | #endif | 492 | #endif |
561 | 493 | ||
@@ -586,8 +518,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, | |||
586 | unsigned long flags) | 518 | unsigned long flags) |
587 | { | 519 | { |
588 | return slice_get_unmapped_area(addr, len, flags, | 520 | return slice_get_unmapped_area(addr, len, flags, |
589 | current->mm->context.user_psize, | 521 | current->mm->context.user_psize, 0); |
590 | 0, 1); | ||
591 | } | 522 | } |
592 | 523 | ||
593 | unsigned long arch_get_unmapped_area_topdown(struct file *filp, | 524 | unsigned long arch_get_unmapped_area_topdown(struct file *filp, |
@@ -597,8 +528,7 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, | |||
597 | const unsigned long flags) | 528 | const unsigned long flags) |
598 | { | 529 | { |
599 | return slice_get_unmapped_area(addr0, len, flags, | 530 | return slice_get_unmapped_area(addr0, len, flags, |
600 | current->mm->context.user_psize, | 531 | current->mm->context.user_psize, 1); |
601 | 1, 1); | ||
602 | } | 532 | } |
603 | 533 | ||
604 | unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr) | 534 | unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr) |
diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 68c57d38745a..0026a37e21fd 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c | |||
@@ -352,7 +352,7 @@ static unsigned long spufs_get_unmapped_area(struct file *file, | |||
352 | 352 | ||
353 | /* Else, try to obtain a 64K pages slice */ | 353 | /* Else, try to obtain a 64K pages slice */ |
354 | return slice_get_unmapped_area(addr, len, flags, | 354 | return slice_get_unmapped_area(addr, len, flags, |
355 | MMU_PAGE_64K, 1, 0); | 355 | MMU_PAGE_64K, 1); |
356 | } | 356 | } |
357 | #endif /* CONFIG_SPU_FS_64K_LS */ | 357 | #endif /* CONFIG_SPU_FS_64K_LS */ |
358 | 358 | ||