1 files changed, 79 insertions, 10 deletions

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 6d7296dd11b8..ef37c85423a5 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1405,12 +1405,25 @@ pgoff_t __basepage_index(struct page *page)
 }
 
 static struct page *alloc_buddy_huge_page(struct hstate *h,
-                gfp_t gfp_mask, int nid, nodemask_t *nmask)
+                gfp_t gfp_mask, int nid, nodemask_t *nmask,
+                nodemask_t *node_alloc_noretry)
 {
         int order = huge_page_order(h);
         struct page *page;
+        bool alloc_try_hard = true;
 
-        gfp_mask |= __GFP_COMP|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
+        /*
+         * By default we always try hard to allocate the page with
+         * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
+         * a loop (to adjust global huge page counts) and previous allocation
+         * failed, do not continue to try hard on the same node.  Use the
+         * node_alloc_noretry bitmap to manage this state information.
+         */
+        if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
+                alloc_try_hard = false;
+        gfp_mask |= __GFP_COMP|__GFP_NOWARN;
+        if (alloc_try_hard)
+                gfp_mask |= __GFP_RETRY_MAYFAIL;
         if (nid == NUMA_NO_NODE)
                 nid = numa_mem_id();
         page = __alloc_pages_nodemask(gfp_mask, order, nid, nmask);
@@ -1419,6 +1432,22 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,
         else
                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
 
+        /*
+         * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
+         * indicates an overall state change.  Clear bit so that we resume
+         * normal 'try hard' allocations.
+         */
+        if (node_alloc_noretry && page && !alloc_try_hard)
+                node_clear(nid, *node_alloc_noretry);
+
+        /*
+         * If we tried hard to get a page but failed, set bit so that
+         * subsequent attempts will not try as hard until there is an
+         * overall state change.
+         */
+        if (node_alloc_noretry && !page && alloc_try_hard)
+                node_set(nid, *node_alloc_noretry);
+
         return page;
 }
 
@@ -1427,7 +1456,8 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,
  * should use this function to get new hugetlb pages
  */
 static struct page *alloc_fresh_huge_page(struct hstate *h,
-                gfp_t gfp_mask, int nid, nodemask_t *nmask)
+                gfp_t gfp_mask, int nid, nodemask_t *nmask,
+                nodemask_t *node_alloc_noretry)
 {
         struct page *page;
 
@@ -1435,7 +1465,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
                 page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
         else
                 page = alloc_buddy_huge_page(h, gfp_mask,
-                                nid, nmask);
+                                nid, nmask, node_alloc_noretry);
         if (!page)
                 return NULL;
 
@@ -1450,14 +1480,16 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
  * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
  * manner.
  */
-static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
+static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
+                                nodemask_t *node_alloc_noretry)
 {
         struct page *page;
         int nr_nodes, node;
         gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
 
         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
-                page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed);
+                page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
+                                                node_alloc_noretry);
                 if (page)
                         break;
         }
@@ -1601,7 +1633,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
                 goto out_unlock;
         spin_unlock(&hugetlb_lock);
 
-        page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask);
+        page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
         if (!page)
                 return NULL;
 
@@ -1637,7 +1669,7 @@ struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
         if (hstate_is_gigantic(h))
                 return NULL;
 
-        page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask);
+        page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
         if (!page)
                 return NULL;
 
@@ -2207,13 +2239,33 @@ static void __init gather_bootmem_prealloc(void)
 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
 {
         unsigned long i;
+        nodemask_t *node_alloc_noretry;
+
+        if (!hstate_is_gigantic(h)) {
+                /*
+                 * Bit mask controlling how hard we retry per-node allocations.
+                 * Ignore errors as lower level routines can deal with
+                 * node_alloc_noretry == NULL.  If this kmalloc fails at boot
+                 * time, we are likely in bigger trouble.
+                 */
+                node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
+                                                GFP_KERNEL);
+        } else {
+                /* allocations done at boot time */
+                node_alloc_noretry = NULL;
+        }
+
+        /* bit mask controlling how hard we retry per-node allocations */
+        if (node_alloc_noretry)
+                nodes_clear(*node_alloc_noretry);
 
         for (i = 0; i < h->max_huge_pages; ++i) {
                 if (hstate_is_gigantic(h)) {
                         if (!alloc_bootmem_huge_page(h))
                                 break;
                 } else if (!alloc_pool_huge_page(h,
-                                         &node_states[N_MEMORY]))
+                                         &node_states[N_MEMORY],
+                                         node_alloc_noretry))
                         break;
                 cond_resched();
         }
@@ -2225,6 +2277,8 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
                         h->max_huge_pages, buf, i);
                 h->max_huge_pages = i;
         }
+
+        kfree(node_alloc_noretry);
 }
 
 static void __init hugetlb_init_hstates(void)
@@ -2323,6 +2377,17 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
                               nodemask_t *nodes_allowed)
 {
         unsigned long min_count, ret;
+        NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
+
+        /*
+         * Bit mask controlling how hard we retry per-node allocations.
+         * If we can not allocate the bit mask, do not attempt to allocate
+         * the requested huge pages.
+         */
+        if (node_alloc_noretry)
+                nodes_clear(*node_alloc_noretry);
+        else
+                return -ENOMEM;
 
         spin_lock(&hugetlb_lock);
 
@@ -2356,6 +2421,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
         if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
                 if (count > persistent_huge_pages(h)) {
                         spin_unlock(&hugetlb_lock);
+                        NODEMASK_FREE(node_alloc_noretry);
                         return -EINVAL;
                 }
                 /* Fall through to decrease pool */
@@ -2388,7 +2454,8 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
                 /* yield cpu to avoid soft lockup */
                 cond_resched();
 
-                ret = alloc_pool_huge_page(h, nodes_allowed);
+                ret = alloc_pool_huge_page(h, nodes_allowed,
+                                                node_alloc_noretry);
                 spin_lock(&hugetlb_lock);
                 if (!ret)
                         goto out;
@@ -2429,6 +2496,8 @@ out:
         h->max_huge_pages = persistent_huge_pages(h);
         spin_unlock(&hugetlb_lock);
 
+        NODEMASK_FREE(node_alloc_noretry);
+
         return 0;
 }