mm/thp: allocate transparent hugepages on local node

This make sure that we try to allocate hugepages from local node if
allowed by mempolicy.  If we can't, we fallback to small page allocation
based on mempolicy.  This is based on the observation that allocating
pages on local node is more beneficial than allocating hugepages on remote
node.

With this patch applied we may find transparent huge page allocation
failures if the current node doesn't have enough freee hugepages.  Before
this patch such failures result in us retrying the allocation on other
nodes in the numa node mask.

[akpm@linux-foundation.org: fix comment, add CONFIG_TRANSPARENT_HUGEPAGE dependency]
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: David Rientjes <rientjes@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 9 insertions, 15 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 889713180980..0531ea7dd7cf 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -761,15 +761,6 @@ static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp)
         return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT)) | extra_gfp;
 }
 
-static inline struct page *alloc_hugepage_vma(int defrag,
-                                              struct vm_area_struct *vma,
-                                              unsigned long haddr, int nd,
-                                              gfp_t extra_gfp)
-{
-        return alloc_pages_vma(alloc_hugepage_gfpmask(defrag, extra_gfp),
-                               HPAGE_PMD_ORDER, vma, haddr, nd);
-}
-
 /* Caller must hold page table lock. */
 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
                 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
@@ -790,6 +781,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                                unsigned long address, pmd_t *pmd,
                                unsigned int flags)
 {
+        gfp_t gfp;
         struct page *page;
         unsigned long haddr = address & HPAGE_PMD_MASK;
 
@@ -824,8 +816,8 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 }
                 return 0;
         }
-        page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
-                        vma, haddr, numa_node_id(), 0);
+        gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
+        page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
         if (unlikely(!page)) {
                 count_vm_event(THP_FAULT_FALLBACK);
                 return VM_FAULT_FALLBACK;
@@ -1113,10 +1105,12 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
         spin_unlock(ptl);
 alloc:
         if (transparent_hugepage_enabled(vma) &&
-            !transparent_hugepage_debug_cow())
-                new_page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
-                                              vma, haddr, numa_node_id(), 0);
-        else
+            !transparent_hugepage_debug_cow()) {
+                gfp_t gfp;
+
+                gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
+                new_page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
+        } else
                 new_page = NULL;
 
         if (unlikely(!new_page)) {