diff options
-rw-r--r-- | mm/huge_memory.c | 83 |
1 files changed, 41 insertions, 42 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 60836870c6f7..6551dd06dd64 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -785,55 +785,54 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
785 | unsigned long haddr = address & HPAGE_PMD_MASK; | 785 | unsigned long haddr = address & HPAGE_PMD_MASK; |
786 | pte_t *pte; | 786 | pte_t *pte; |
787 | 787 | ||
788 | if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) { | 788 | if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) |
789 | if (unlikely(anon_vma_prepare(vma))) | 789 | goto out; |
790 | return VM_FAULT_OOM; | 790 | if (unlikely(anon_vma_prepare(vma))) |
791 | if (unlikely(khugepaged_enter(vma))) | 791 | return VM_FAULT_OOM; |
792 | if (unlikely(khugepaged_enter(vma))) | ||
793 | return VM_FAULT_OOM; | ||
794 | if (!(flags & FAULT_FLAG_WRITE) && | ||
795 | transparent_hugepage_use_zero_page()) { | ||
796 | pgtable_t pgtable; | ||
797 | struct page *zero_page; | ||
798 | bool set; | ||
799 | pgtable = pte_alloc_one(mm, haddr); | ||
800 | if (unlikely(!pgtable)) | ||
792 | return VM_FAULT_OOM; | 801 | return VM_FAULT_OOM; |
793 | if (!(flags & FAULT_FLAG_WRITE) && | 802 | zero_page = get_huge_zero_page(); |
794 | transparent_hugepage_use_zero_page()) { | 803 | if (unlikely(!zero_page)) { |
795 | pgtable_t pgtable; | 804 | pte_free(mm, pgtable); |
796 | struct page *zero_page; | ||
797 | bool set; | ||
798 | pgtable = pte_alloc_one(mm, haddr); | ||
799 | if (unlikely(!pgtable)) | ||
800 | return VM_FAULT_OOM; | ||
801 | zero_page = get_huge_zero_page(); | ||
802 | if (unlikely(!zero_page)) { | ||
803 | pte_free(mm, pgtable); | ||
804 | count_vm_event(THP_FAULT_FALLBACK); | ||
805 | goto out; | ||
806 | } | ||
807 | spin_lock(&mm->page_table_lock); | ||
808 | set = set_huge_zero_page(pgtable, mm, vma, haddr, pmd, | ||
809 | zero_page); | ||
810 | spin_unlock(&mm->page_table_lock); | ||
811 | if (!set) { | ||
812 | pte_free(mm, pgtable); | ||
813 | put_huge_zero_page(); | ||
814 | } | ||
815 | return 0; | ||
816 | } | ||
817 | page = alloc_hugepage_vma(transparent_hugepage_defrag(vma), | ||
818 | vma, haddr, numa_node_id(), 0); | ||
819 | if (unlikely(!page)) { | ||
820 | count_vm_event(THP_FAULT_FALLBACK); | 805 | count_vm_event(THP_FAULT_FALLBACK); |
821 | goto out; | 806 | goto out; |
822 | } | 807 | } |
823 | count_vm_event(THP_FAULT_ALLOC); | 808 | spin_lock(&mm->page_table_lock); |
824 | if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) { | 809 | set = set_huge_zero_page(pgtable, mm, vma, haddr, pmd, |
825 | put_page(page); | 810 | zero_page); |
826 | goto out; | 811 | spin_unlock(&mm->page_table_lock); |
827 | } | 812 | if (!set) { |
828 | if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, | 813 | pte_free(mm, pgtable); |
829 | page))) { | 814 | put_huge_zero_page(); |
830 | mem_cgroup_uncharge_page(page); | ||
831 | put_page(page); | ||
832 | goto out; | ||
833 | } | 815 | } |
834 | |||
835 | return 0; | 816 | return 0; |
836 | } | 817 | } |
818 | page = alloc_hugepage_vma(transparent_hugepage_defrag(vma), | ||
819 | vma, haddr, numa_node_id(), 0); | ||
820 | if (unlikely(!page)) { | ||
821 | count_vm_event(THP_FAULT_FALLBACK); | ||
822 | goto out; | ||
823 | } | ||
824 | count_vm_event(THP_FAULT_ALLOC); | ||
825 | if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) { | ||
826 | put_page(page); | ||
827 | goto out; | ||
828 | } | ||
829 | if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page))) { | ||
830 | mem_cgroup_uncharge_page(page); | ||
831 | put_page(page); | ||
832 | goto out; | ||
833 | } | ||
834 | |||
835 | return 0; | ||
837 | out: | 836 | out: |
838 | /* | 837 | /* |
839 | * Use __pte_alloc instead of pte_alloc_map, because we can't | 838 | * Use __pte_alloc instead of pte_alloc_map, because we can't |