diff options
Diffstat (limited to 'mm/huge_memory.c')
-rw-r--r-- | mm/huge_memory.c | 29 |
1 files changed, 19 insertions, 10 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index f0e5306eeb55..57c4b9309015 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -636,16 +636,12 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, | |||
636 | unsigned long haddr, pmd_t *pmd, | 636 | unsigned long haddr, pmd_t *pmd, |
637 | struct page *page) | 637 | struct page *page) |
638 | { | 638 | { |
639 | int ret = 0; | ||
640 | pgtable_t pgtable; | 639 | pgtable_t pgtable; |
641 | 640 | ||
642 | VM_BUG_ON(!PageCompound(page)); | 641 | VM_BUG_ON(!PageCompound(page)); |
643 | pgtable = pte_alloc_one(mm, haddr); | 642 | pgtable = pte_alloc_one(mm, haddr); |
644 | if (unlikely(!pgtable)) { | 643 | if (unlikely(!pgtable)) |
645 | mem_cgroup_uncharge_page(page); | ||
646 | put_page(page); | ||
647 | return VM_FAULT_OOM; | 644 | return VM_FAULT_OOM; |
648 | } | ||
649 | 645 | ||
650 | clear_huge_page(page, haddr, HPAGE_PMD_NR); | 646 | clear_huge_page(page, haddr, HPAGE_PMD_NR); |
651 | __SetPageUptodate(page); | 647 | __SetPageUptodate(page); |
@@ -675,7 +671,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, | |||
675 | spin_unlock(&mm->page_table_lock); | 671 | spin_unlock(&mm->page_table_lock); |
676 | } | 672 | } |
677 | 673 | ||
678 | return ret; | 674 | return 0; |
679 | } | 675 | } |
680 | 676 | ||
681 | static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp) | 677 | static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp) |
@@ -724,8 +720,14 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
724 | put_page(page); | 720 | put_page(page); |
725 | goto out; | 721 | goto out; |
726 | } | 722 | } |
723 | if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, | ||
724 | page))) { | ||
725 | mem_cgroup_uncharge_page(page); | ||
726 | put_page(page); | ||
727 | goto out; | ||
728 | } | ||
727 | 729 | ||
728 | return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page); | 730 | return 0; |
729 | } | 731 | } |
730 | out: | 732 | out: |
731 | /* | 733 | /* |
@@ -950,6 +952,8 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
950 | count_vm_event(THP_FAULT_FALLBACK); | 952 | count_vm_event(THP_FAULT_FALLBACK); |
951 | ret = do_huge_pmd_wp_page_fallback(mm, vma, address, | 953 | ret = do_huge_pmd_wp_page_fallback(mm, vma, address, |
952 | pmd, orig_pmd, page, haddr); | 954 | pmd, orig_pmd, page, haddr); |
955 | if (ret & VM_FAULT_OOM) | ||
956 | split_huge_page(page); | ||
953 | put_page(page); | 957 | put_page(page); |
954 | goto out; | 958 | goto out; |
955 | } | 959 | } |
@@ -957,6 +961,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
957 | 961 | ||
958 | if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) { | 962 | if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) { |
959 | put_page(new_page); | 963 | put_page(new_page); |
964 | split_huge_page(page); | ||
960 | put_page(page); | 965 | put_page(page); |
961 | ret |= VM_FAULT_OOM; | 966 | ret |= VM_FAULT_OOM; |
962 | goto out; | 967 | goto out; |
@@ -968,8 +973,10 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
968 | spin_lock(&mm->page_table_lock); | 973 | spin_lock(&mm->page_table_lock); |
969 | put_page(page); | 974 | put_page(page); |
970 | if (unlikely(!pmd_same(*pmd, orig_pmd))) { | 975 | if (unlikely(!pmd_same(*pmd, orig_pmd))) { |
976 | spin_unlock(&mm->page_table_lock); | ||
971 | mem_cgroup_uncharge_page(new_page); | 977 | mem_cgroup_uncharge_page(new_page); |
972 | put_page(new_page); | 978 | put_page(new_page); |
979 | goto out; | ||
973 | } else { | 980 | } else { |
974 | pmd_t entry; | 981 | pmd_t entry; |
975 | VM_BUG_ON(!PageHead(page)); | 982 | VM_BUG_ON(!PageHead(page)); |
@@ -1224,10 +1231,13 @@ static void __split_huge_page_refcount(struct page *page) | |||
1224 | { | 1231 | { |
1225 | int i; | 1232 | int i; |
1226 | struct zone *zone = page_zone(page); | 1233 | struct zone *zone = page_zone(page); |
1234 | struct lruvec *lruvec; | ||
1227 | int tail_count = 0; | 1235 | int tail_count = 0; |
1228 | 1236 | ||
1229 | /* prevent PageLRU to go away from under us, and freeze lru stats */ | 1237 | /* prevent PageLRU to go away from under us, and freeze lru stats */ |
1230 | spin_lock_irq(&zone->lru_lock); | 1238 | spin_lock_irq(&zone->lru_lock); |
1239 | lruvec = mem_cgroup_page_lruvec(page, zone); | ||
1240 | |||
1231 | compound_lock(page); | 1241 | compound_lock(page); |
1232 | /* complete memcg works before add pages to LRU */ | 1242 | /* complete memcg works before add pages to LRU */ |
1233 | mem_cgroup_split_huge_fixup(page); | 1243 | mem_cgroup_split_huge_fixup(page); |
@@ -1302,13 +1312,12 @@ static void __split_huge_page_refcount(struct page *page) | |||
1302 | BUG_ON(!PageDirty(page_tail)); | 1312 | BUG_ON(!PageDirty(page_tail)); |
1303 | BUG_ON(!PageSwapBacked(page_tail)); | 1313 | BUG_ON(!PageSwapBacked(page_tail)); |
1304 | 1314 | ||
1305 | 1315 | lru_add_page_tail(page, page_tail, lruvec); | |
1306 | lru_add_page_tail(zone, page, page_tail); | ||
1307 | } | 1316 | } |
1308 | atomic_sub(tail_count, &page->_count); | 1317 | atomic_sub(tail_count, &page->_count); |
1309 | BUG_ON(atomic_read(&page->_count) <= 0); | 1318 | BUG_ON(atomic_read(&page->_count) <= 0); |
1310 | 1319 | ||
1311 | __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); | 1320 | __mod_zone_page_state(zone, NR_ANON_TRANSPARENT_HUGEPAGES, -1); |
1312 | __mod_zone_page_state(zone, NR_ANON_PAGES, HPAGE_PMD_NR); | 1321 | __mod_zone_page_state(zone, NR_ANON_PAGES, HPAGE_PMD_NR); |
1313 | 1322 | ||
1314 | ClearPageCompound(page); | 1323 | ClearPageCompound(page); |