diff options
Diffstat (limited to 'mm/vmscan.c')
-rw-r--r-- | mm/vmscan.c | 491 |
1 files changed, 2 insertions, 489 deletions
diff --git a/mm/vmscan.c b/mm/vmscan.c index 548e023c193b..fd572bbdc9f5 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c | |||
@@ -42,18 +42,6 @@ | |||
42 | 42 | ||
43 | #include "internal.h" | 43 | #include "internal.h" |
44 | 44 | ||
45 | /* possible outcome of pageout() */ | ||
46 | typedef enum { | ||
47 | /* failed to write page out, page is locked */ | ||
48 | PAGE_KEEP, | ||
49 | /* move page to the active list, page is locked */ | ||
50 | PAGE_ACTIVATE, | ||
51 | /* page has been sent to the disk successfully, page is unlocked */ | ||
52 | PAGE_SUCCESS, | ||
53 | /* page is clean and locked */ | ||
54 | PAGE_CLEAN, | ||
55 | } pageout_t; | ||
56 | |||
57 | struct scan_control { | 45 | struct scan_control { |
58 | /* Incremented by the number of inactive pages that were scanned */ | 46 | /* Incremented by the number of inactive pages that were scanned */ |
59 | unsigned long nr_scanned; | 47 | unsigned long nr_scanned; |
@@ -304,7 +292,7 @@ static void handle_write_error(struct address_space *mapping, | |||
304 | * pageout is called by shrink_page_list() for each dirty page. | 292 | * pageout is called by shrink_page_list() for each dirty page. |
305 | * Calls ->writepage(). | 293 | * Calls ->writepage(). |
306 | */ | 294 | */ |
307 | static pageout_t pageout(struct page *page, struct address_space *mapping) | 295 | pageout_t pageout(struct page *page, struct address_space *mapping) |
308 | { | 296 | { |
309 | /* | 297 | /* |
310 | * If the page is dirty, only perform writeback if that write | 298 | * If the page is dirty, only perform writeback if that write |
@@ -372,7 +360,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping) | |||
372 | return PAGE_CLEAN; | 360 | return PAGE_CLEAN; |
373 | } | 361 | } |
374 | 362 | ||
375 | static int remove_mapping(struct address_space *mapping, struct page *page) | 363 | int remove_mapping(struct address_space *mapping, struct page *page) |
376 | { | 364 | { |
377 | if (!mapping) | 365 | if (!mapping) |
378 | return 0; /* truncate got there first */ | 366 | return 0; /* truncate got there first */ |
@@ -570,481 +558,6 @@ keep: | |||
570 | return nr_reclaimed; | 558 | return nr_reclaimed; |
571 | } | 559 | } |
572 | 560 | ||
573 | #ifdef CONFIG_MIGRATION | ||
574 | static inline void move_to_lru(struct page *page) | ||
575 | { | ||
576 | list_del(&page->lru); | ||
577 | if (PageActive(page)) { | ||
578 | /* | ||
579 | * lru_cache_add_active checks that | ||
580 | * the PG_active bit is off. | ||
581 | */ | ||
582 | ClearPageActive(page); | ||
583 | lru_cache_add_active(page); | ||
584 | } else { | ||
585 | lru_cache_add(page); | ||
586 | } | ||
587 | put_page(page); | ||
588 | } | ||
589 | |||
590 | /* | ||
591 | * Add isolated pages on the list back to the LRU. | ||
592 | * | ||
593 | * returns the number of pages put back. | ||
594 | */ | ||
595 | unsigned long putback_lru_pages(struct list_head *l) | ||
596 | { | ||
597 | struct page *page; | ||
598 | struct page *page2; | ||
599 | unsigned long count = 0; | ||
600 | |||
601 | list_for_each_entry_safe(page, page2, l, lru) { | ||
602 | move_to_lru(page); | ||
603 | count++; | ||
604 | } | ||
605 | return count; | ||
606 | } | ||
607 | |||
608 | /* | ||
609 | * Non migratable page | ||
610 | */ | ||
611 | int fail_migrate_page(struct page *newpage, struct page *page) | ||
612 | { | ||
613 | return -EIO; | ||
614 | } | ||
615 | EXPORT_SYMBOL(fail_migrate_page); | ||
616 | |||
617 | /* | ||
618 | * swapout a single page | ||
619 | * page is locked upon entry, unlocked on exit | ||
620 | */ | ||
621 | static int swap_page(struct page *page) | ||
622 | { | ||
623 | struct address_space *mapping = page_mapping(page); | ||
624 | |||
625 | if (page_mapped(page) && mapping) | ||
626 | if (try_to_unmap(page, 1) != SWAP_SUCCESS) | ||
627 | goto unlock_retry; | ||
628 | |||
629 | if (PageDirty(page)) { | ||
630 | /* Page is dirty, try to write it out here */ | ||
631 | switch(pageout(page, mapping)) { | ||
632 | case PAGE_KEEP: | ||
633 | case PAGE_ACTIVATE: | ||
634 | goto unlock_retry; | ||
635 | |||
636 | case PAGE_SUCCESS: | ||
637 | goto retry; | ||
638 | |||
639 | case PAGE_CLEAN: | ||
640 | ; /* try to free the page below */ | ||
641 | } | ||
642 | } | ||
643 | |||
644 | if (PagePrivate(page)) { | ||
645 | if (!try_to_release_page(page, GFP_KERNEL) || | ||
646 | (!mapping && page_count(page) == 1)) | ||
647 | goto unlock_retry; | ||
648 | } | ||
649 | |||
650 | if (remove_mapping(mapping, page)) { | ||
651 | /* Success */ | ||
652 | unlock_page(page); | ||
653 | return 0; | ||
654 | } | ||
655 | |||
656 | unlock_retry: | ||
657 | unlock_page(page); | ||
658 | |||
659 | retry: | ||
660 | return -EAGAIN; | ||
661 | } | ||
662 | EXPORT_SYMBOL(swap_page); | ||
663 | |||
664 | /* | ||
665 | * Page migration was first developed in the context of the memory hotplug | ||
666 | * project. The main authors of the migration code are: | ||
667 | * | ||
668 | * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> | ||
669 | * Hirokazu Takahashi <taka@valinux.co.jp> | ||
670 | * Dave Hansen <haveblue@us.ibm.com> | ||
671 | * Christoph Lameter <clameter@sgi.com> | ||
672 | */ | ||
673 | |||
674 | /* | ||
675 | * Remove references for a page and establish the new page with the correct | ||
676 | * basic settings to be able to stop accesses to the page. | ||
677 | */ | ||
678 | int migrate_page_remove_references(struct page *newpage, | ||
679 | struct page *page, int nr_refs) | ||
680 | { | ||
681 | struct address_space *mapping = page_mapping(page); | ||
682 | struct page **radix_pointer; | ||
683 | |||
684 | /* | ||
685 | * Avoid doing any of the following work if the page count | ||
686 | * indicates that the page is in use or truncate has removed | ||
687 | * the page. | ||
688 | */ | ||
689 | if (!mapping || page_mapcount(page) + nr_refs != page_count(page)) | ||
690 | return -EAGAIN; | ||
691 | |||
692 | /* | ||
693 | * Establish swap ptes for anonymous pages or destroy pte | ||
694 | * maps for files. | ||
695 | * | ||
696 | * In order to reestablish file backed mappings the fault handlers | ||
697 | * will take the radix tree_lock which may then be used to stop | ||
698 | * processses from accessing this page until the new page is ready. | ||
699 | * | ||
700 | * A process accessing via a swap pte (an anonymous page) will take a | ||
701 | * page_lock on the old page which will block the process until the | ||
702 | * migration attempt is complete. At that time the PageSwapCache bit | ||
703 | * will be examined. If the page was migrated then the PageSwapCache | ||
704 | * bit will be clear and the operation to retrieve the page will be | ||
705 | * retried which will find the new page in the radix tree. Then a new | ||
706 | * direct mapping may be generated based on the radix tree contents. | ||
707 | * | ||
708 | * If the page was not migrated then the PageSwapCache bit | ||
709 | * is still set and the operation may continue. | ||
710 | */ | ||
711 | if (try_to_unmap(page, 1) == SWAP_FAIL) | ||
712 | /* A vma has VM_LOCKED set -> Permanent failure */ | ||
713 | return -EPERM; | ||
714 | |||
715 | /* | ||
716 | * Give up if we were unable to remove all mappings. | ||
717 | */ | ||
718 | if (page_mapcount(page)) | ||
719 | return -EAGAIN; | ||
720 | |||
721 | write_lock_irq(&mapping->tree_lock); | ||
722 | |||
723 | radix_pointer = (struct page **)radix_tree_lookup_slot( | ||
724 | &mapping->page_tree, | ||
725 | page_index(page)); | ||
726 | |||
727 | if (!page_mapping(page) || page_count(page) != nr_refs || | ||
728 | *radix_pointer != page) { | ||
729 | write_unlock_irq(&mapping->tree_lock); | ||
730 | return -EAGAIN; | ||
731 | } | ||
732 | |||
733 | /* | ||
734 | * Now we know that no one else is looking at the page. | ||
735 | * | ||
736 | * Certain minimal information about a page must be available | ||
737 | * in order for other subsystems to properly handle the page if they | ||
738 | * find it through the radix tree update before we are finished | ||
739 | * copying the page. | ||
740 | */ | ||
741 | get_page(newpage); | ||
742 | newpage->index = page->index; | ||
743 | newpage->mapping = page->mapping; | ||
744 | if (PageSwapCache(page)) { | ||
745 | SetPageSwapCache(newpage); | ||
746 | set_page_private(newpage, page_private(page)); | ||
747 | } | ||
748 | |||
749 | *radix_pointer = newpage; | ||
750 | __put_page(page); | ||
751 | write_unlock_irq(&mapping->tree_lock); | ||
752 | |||
753 | return 0; | ||
754 | } | ||
755 | EXPORT_SYMBOL(migrate_page_remove_references); | ||
756 | |||
757 | /* | ||
758 | * Copy the page to its new location | ||
759 | */ | ||
760 | void migrate_page_copy(struct page *newpage, struct page *page) | ||
761 | { | ||
762 | copy_highpage(newpage, page); | ||
763 | |||
764 | if (PageError(page)) | ||
765 | SetPageError(newpage); | ||
766 | if (PageReferenced(page)) | ||
767 | SetPageReferenced(newpage); | ||
768 | if (PageUptodate(page)) | ||
769 | SetPageUptodate(newpage); | ||
770 | if (PageActive(page)) | ||
771 | SetPageActive(newpage); | ||
772 | if (PageChecked(page)) | ||
773 | SetPageChecked(newpage); | ||
774 | if (PageMappedToDisk(page)) | ||
775 | SetPageMappedToDisk(newpage); | ||
776 | |||
777 | if (PageDirty(page)) { | ||
778 | clear_page_dirty_for_io(page); | ||
779 | set_page_dirty(newpage); | ||
780 | } | ||
781 | |||
782 | ClearPageSwapCache(page); | ||
783 | ClearPageActive(page); | ||
784 | ClearPagePrivate(page); | ||
785 | set_page_private(page, 0); | ||
786 | page->mapping = NULL; | ||
787 | |||
788 | /* | ||
789 | * If any waiters have accumulated on the new page then | ||
790 | * wake them up. | ||
791 | */ | ||
792 | if (PageWriteback(newpage)) | ||
793 | end_page_writeback(newpage); | ||
794 | } | ||
795 | EXPORT_SYMBOL(migrate_page_copy); | ||
796 | |||
797 | /* | ||
798 | * Common logic to directly migrate a single page suitable for | ||
799 | * pages that do not use PagePrivate. | ||
800 | * | ||
801 | * Pages are locked upon entry and exit. | ||
802 | */ | ||
803 | int migrate_page(struct page *newpage, struct page *page) | ||
804 | { | ||
805 | int rc; | ||
806 | |||
807 | BUG_ON(PageWriteback(page)); /* Writeback must be complete */ | ||
808 | |||
809 | rc = migrate_page_remove_references(newpage, page, 2); | ||
810 | |||
811 | if (rc) | ||
812 | return rc; | ||
813 | |||
814 | migrate_page_copy(newpage, page); | ||
815 | |||
816 | /* | ||
817 | * Remove auxiliary swap entries and replace | ||
818 | * them with real ptes. | ||
819 | * | ||
820 | * Note that a real pte entry will allow processes that are not | ||
821 | * waiting on the page lock to use the new page via the page tables | ||
822 | * before the new page is unlocked. | ||
823 | */ | ||
824 | remove_from_swap(newpage); | ||
825 | return 0; | ||
826 | } | ||
827 | EXPORT_SYMBOL(migrate_page); | ||
828 | |||
829 | /* | ||
830 | * migrate_pages | ||
831 | * | ||
832 | * Two lists are passed to this function. The first list | ||
833 | * contains the pages isolated from the LRU to be migrated. | ||
834 | * The second list contains new pages that the pages isolated | ||
835 | * can be moved to. If the second list is NULL then all | ||
836 | * pages are swapped out. | ||
837 | * | ||
838 | * The function returns after 10 attempts or if no pages | ||
839 | * are movable anymore because to has become empty | ||
840 | * or no retryable pages exist anymore. | ||
841 | * | ||
842 | * Return: Number of pages not migrated when "to" ran empty. | ||
843 | */ | ||
844 | unsigned long migrate_pages(struct list_head *from, struct list_head *to, | ||
845 | struct list_head *moved, struct list_head *failed) | ||
846 | { | ||
847 | unsigned long retry; | ||
848 | unsigned long nr_failed = 0; | ||
849 | int pass = 0; | ||
850 | struct page *page; | ||
851 | struct page *page2; | ||
852 | int swapwrite = current->flags & PF_SWAPWRITE; | ||
853 | int rc; | ||
854 | |||
855 | if (!swapwrite) | ||
856 | current->flags |= PF_SWAPWRITE; | ||
857 | |||
858 | redo: | ||
859 | retry = 0; | ||
860 | |||
861 | list_for_each_entry_safe(page, page2, from, lru) { | ||
862 | struct page *newpage = NULL; | ||
863 | struct address_space *mapping; | ||
864 | |||
865 | cond_resched(); | ||
866 | |||
867 | rc = 0; | ||
868 | if (page_count(page) == 1) | ||
869 | /* page was freed from under us. So we are done. */ | ||
870 | goto next; | ||
871 | |||
872 | if (to && list_empty(to)) | ||
873 | break; | ||
874 | |||
875 | /* | ||
876 | * Skip locked pages during the first two passes to give the | ||
877 | * functions holding the lock time to release the page. Later we | ||
878 | * use lock_page() to have a higher chance of acquiring the | ||
879 | * lock. | ||
880 | */ | ||
881 | rc = -EAGAIN; | ||
882 | if (pass > 2) | ||
883 | lock_page(page); | ||
884 | else | ||
885 | if (TestSetPageLocked(page)) | ||
886 | goto next; | ||
887 | |||
888 | /* | ||
889 | * Only wait on writeback if we have already done a pass where | ||
890 | * we we may have triggered writeouts for lots of pages. | ||
891 | */ | ||
892 | if (pass > 0) { | ||
893 | wait_on_page_writeback(page); | ||
894 | } else { | ||
895 | if (PageWriteback(page)) | ||
896 | goto unlock_page; | ||
897 | } | ||
898 | |||
899 | /* | ||
900 | * Anonymous pages must have swap cache references otherwise | ||
901 | * the information contained in the page maps cannot be | ||
902 | * preserved. | ||
903 | */ | ||
904 | if (PageAnon(page) && !PageSwapCache(page)) { | ||
905 | if (!add_to_swap(page, GFP_KERNEL)) { | ||
906 | rc = -ENOMEM; | ||
907 | goto unlock_page; | ||
908 | } | ||
909 | } | ||
910 | |||
911 | if (!to) { | ||
912 | rc = swap_page(page); | ||
913 | goto next; | ||
914 | } | ||
915 | |||
916 | newpage = lru_to_page(to); | ||
917 | lock_page(newpage); | ||
918 | |||
919 | /* | ||
920 | * Pages are properly locked and writeback is complete. | ||
921 | * Try to migrate the page. | ||
922 | */ | ||
923 | mapping = page_mapping(page); | ||
924 | if (!mapping) | ||
925 | goto unlock_both; | ||
926 | |||
927 | if (mapping->a_ops->migratepage) { | ||
928 | /* | ||
929 | * Most pages have a mapping and most filesystems | ||
930 | * should provide a migration function. Anonymous | ||
931 | * pages are part of swap space which also has its | ||
932 | * own migration function. This is the most common | ||
933 | * path for page migration. | ||
934 | */ | ||
935 | rc = mapping->a_ops->migratepage(newpage, page); | ||
936 | goto unlock_both; | ||
937 | } | ||
938 | |||
939 | /* | ||
940 | * Default handling if a filesystem does not provide | ||
941 | * a migration function. We can only migrate clean | ||
942 | * pages so try to write out any dirty pages first. | ||
943 | */ | ||
944 | if (PageDirty(page)) { | ||
945 | switch (pageout(page, mapping)) { | ||
946 | case PAGE_KEEP: | ||
947 | case PAGE_ACTIVATE: | ||
948 | goto unlock_both; | ||
949 | |||
950 | case PAGE_SUCCESS: | ||
951 | unlock_page(newpage); | ||
952 | goto next; | ||
953 | |||
954 | case PAGE_CLEAN: | ||
955 | ; /* try to migrate the page below */ | ||
956 | } | ||
957 | } | ||
958 | |||
959 | /* | ||
960 | * Buffers are managed in a filesystem specific way. | ||
961 | * We must have no buffers or drop them. | ||
962 | */ | ||
963 | if (!page_has_buffers(page) || | ||
964 | try_to_release_page(page, GFP_KERNEL)) { | ||
965 | rc = migrate_page(newpage, page); | ||
966 | goto unlock_both; | ||
967 | } | ||
968 | |||
969 | /* | ||
970 | * On early passes with mapped pages simply | ||
971 | * retry. There may be a lock held for some | ||
972 | * buffers that may go away. Later | ||
973 | * swap them out. | ||
974 | */ | ||
975 | if (pass > 4) { | ||
976 | /* | ||
977 | * Persistently unable to drop buffers..... As a | ||
978 | * measure of last resort we fall back to | ||
979 | * swap_page(). | ||
980 | */ | ||
981 | unlock_page(newpage); | ||
982 | newpage = NULL; | ||
983 | rc = swap_page(page); | ||
984 | goto next; | ||
985 | } | ||
986 | |||
987 | unlock_both: | ||
988 | unlock_page(newpage); | ||
989 | |||
990 | unlock_page: | ||
991 | unlock_page(page); | ||
992 | |||
993 | next: | ||
994 | if (rc == -EAGAIN) { | ||
995 | retry++; | ||
996 | } else if (rc) { | ||
997 | /* Permanent failure */ | ||
998 | list_move(&page->lru, failed); | ||
999 | nr_failed++; | ||
1000 | } else { | ||
1001 | if (newpage) { | ||
1002 | /* Successful migration. Return page to LRU */ | ||
1003 | move_to_lru(newpage); | ||
1004 | } | ||
1005 | list_move(&page->lru, moved); | ||
1006 | } | ||
1007 | } | ||
1008 | if (retry && pass++ < 10) | ||
1009 | goto redo; | ||
1010 | |||
1011 | if (!swapwrite) | ||
1012 | current->flags &= ~PF_SWAPWRITE; | ||
1013 | |||
1014 | return nr_failed + retry; | ||
1015 | } | ||
1016 | |||
1017 | /* | ||
1018 | * Isolate one page from the LRU lists and put it on the | ||
1019 | * indicated list with elevated refcount. | ||
1020 | * | ||
1021 | * Result: | ||
1022 | * 0 = page not on LRU list | ||
1023 | * 1 = page removed from LRU list and added to the specified list. | ||
1024 | */ | ||
1025 | int isolate_lru_page(struct page *page) | ||
1026 | { | ||
1027 | int ret = 0; | ||
1028 | |||
1029 | if (PageLRU(page)) { | ||
1030 | struct zone *zone = page_zone(page); | ||
1031 | spin_lock_irq(&zone->lru_lock); | ||
1032 | if (PageLRU(page)) { | ||
1033 | ret = 1; | ||
1034 | get_page(page); | ||
1035 | ClearPageLRU(page); | ||
1036 | if (PageActive(page)) | ||
1037 | del_page_from_active_list(zone, page); | ||
1038 | else | ||
1039 | del_page_from_inactive_list(zone, page); | ||
1040 | } | ||
1041 | spin_unlock_irq(&zone->lru_lock); | ||
1042 | } | ||
1043 | |||
1044 | return ret; | ||
1045 | } | ||
1046 | #endif | ||
1047 | |||
1048 | /* | 561 | /* |
1049 | * zone->lru_lock is heavily contended. Some of the functions that | 562 | * zone->lru_lock is heavily contended. Some of the functions that |
1050 | * shrink the lists perform better by taking out a batch of pages | 563 | * shrink the lists perform better by taking out a batch of pages |