diff options
Diffstat (limited to 'mm/swap.c')
-rw-r--r-- | mm/swap.c | 34 |
1 files changed, 21 insertions, 13 deletions
@@ -31,6 +31,7 @@ | |||
31 | #include <linux/memcontrol.h> | 31 | #include <linux/memcontrol.h> |
32 | #include <linux/gfp.h> | 32 | #include <linux/gfp.h> |
33 | #include <linux/uio.h> | 33 | #include <linux/uio.h> |
34 | #include <linux/hugetlb.h> | ||
34 | 35 | ||
35 | #include "internal.h" | 36 | #include "internal.h" |
36 | 37 | ||
@@ -42,7 +43,7 @@ int page_cluster; | |||
42 | 43 | ||
43 | static DEFINE_PER_CPU(struct pagevec, lru_add_pvec); | 44 | static DEFINE_PER_CPU(struct pagevec, lru_add_pvec); |
44 | static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); | 45 | static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); |
45 | static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs); | 46 | static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs); |
46 | 47 | ||
47 | /* | 48 | /* |
48 | * This path almost never happens for VM activity - pages are normally | 49 | * This path almost never happens for VM activity - pages are normally |
@@ -75,7 +76,14 @@ static void __put_compound_page(struct page *page) | |||
75 | { | 76 | { |
76 | compound_page_dtor *dtor; | 77 | compound_page_dtor *dtor; |
77 | 78 | ||
78 | __page_cache_release(page); | 79 | /* |
80 | * __page_cache_release() is supposed to be called for thp, not for | ||
81 | * hugetlb. This is because hugetlb page does never have PageLRU set | ||
82 | * (it's never listed to any LRU lists) and no memcg routines should | ||
83 | * be called for hugetlb (it has a separate hugetlb_cgroup.) | ||
84 | */ | ||
85 | if (!PageHuge(page)) | ||
86 | __page_cache_release(page); | ||
79 | dtor = get_compound_page_dtor(page); | 87 | dtor = get_compound_page_dtor(page); |
80 | (*dtor)(page); | 88 | (*dtor)(page); |
81 | } | 89 | } |
@@ -743,7 +751,7 @@ void lru_cache_add_active_or_unevictable(struct page *page, | |||
743 | * be write it out by flusher threads as this is much more effective | 751 | * be write it out by flusher threads as this is much more effective |
744 | * than the single-page writeout from reclaim. | 752 | * than the single-page writeout from reclaim. |
745 | */ | 753 | */ |
746 | static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, | 754 | static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, |
747 | void *arg) | 755 | void *arg) |
748 | { | 756 | { |
749 | int lru, file; | 757 | int lru, file; |
@@ -811,36 +819,36 @@ void lru_add_drain_cpu(int cpu) | |||
811 | local_irq_restore(flags); | 819 | local_irq_restore(flags); |
812 | } | 820 | } |
813 | 821 | ||
814 | pvec = &per_cpu(lru_deactivate_pvecs, cpu); | 822 | pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); |
815 | if (pagevec_count(pvec)) | 823 | if (pagevec_count(pvec)) |
816 | pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); | 824 | pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); |
817 | 825 | ||
818 | activate_page_drain(cpu); | 826 | activate_page_drain(cpu); |
819 | } | 827 | } |
820 | 828 | ||
821 | /** | 829 | /** |
822 | * deactivate_page - forcefully deactivate a page | 830 | * deactivate_file_page - forcefully deactivate a file page |
823 | * @page: page to deactivate | 831 | * @page: page to deactivate |
824 | * | 832 | * |
825 | * This function hints the VM that @page is a good reclaim candidate, | 833 | * This function hints the VM that @page is a good reclaim candidate, |
826 | * for example if its invalidation fails due to the page being dirty | 834 | * for example if its invalidation fails due to the page being dirty |
827 | * or under writeback. | 835 | * or under writeback. |
828 | */ | 836 | */ |
829 | void deactivate_page(struct page *page) | 837 | void deactivate_file_page(struct page *page) |
830 | { | 838 | { |
831 | /* | 839 | /* |
832 | * In a workload with many unevictable page such as mprotect, unevictable | 840 | * In a workload with many unevictable page such as mprotect, |
833 | * page deactivation for accelerating reclaim is pointless. | 841 | * unevictable page deactivation for accelerating reclaim is pointless. |
834 | */ | 842 | */ |
835 | if (PageUnevictable(page)) | 843 | if (PageUnevictable(page)) |
836 | return; | 844 | return; |
837 | 845 | ||
838 | if (likely(get_page_unless_zero(page))) { | 846 | if (likely(get_page_unless_zero(page))) { |
839 | struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs); | 847 | struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs); |
840 | 848 | ||
841 | if (!pagevec_add(pvec, page)) | 849 | if (!pagevec_add(pvec, page)) |
842 | pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); | 850 | pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); |
843 | put_cpu_var(lru_deactivate_pvecs); | 851 | put_cpu_var(lru_deactivate_file_pvecs); |
844 | } | 852 | } |
845 | } | 853 | } |
846 | 854 | ||
@@ -872,7 +880,7 @@ void lru_add_drain_all(void) | |||
872 | 880 | ||
873 | if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || | 881 | if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || |
874 | pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) || | 882 | pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) || |
875 | pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) || | 883 | pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) || |
876 | need_activate_page_drain(cpu)) { | 884 | need_activate_page_drain(cpu)) { |
877 | INIT_WORK(work, lru_add_drain_per_cpu); | 885 | INIT_WORK(work, lru_add_drain_per_cpu); |
878 | schedule_work_on(cpu, work); | 886 | schedule_work_on(cpu, work); |