1 files changed, 67 insertions, 39 deletions

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index e83ad2c9228c..a56e6f3ce979 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -578,41 +578,6 @@ static void free_huge_page(struct page *page)
                 hugetlb_put_quota(mapping, 1);
 }
 
-/*
- * Increment or decrement surplus_huge_pages.  Keep node-specific counters
- * balanced by operating on them in a round-robin fashion.
- * Returns 1 if an adjustment was made.
- */
-static int adjust_pool_surplus(struct hstate *h, int delta)
-{
-        static int prev_nid;
-        int nid = prev_nid;
-        int ret = 0;
-
-        VM_BUG_ON(delta != -1 && delta != 1);
-        do {
-                nid = next_node(nid, node_online_map);
-                if (nid == MAX_NUMNODES)
-                        nid = first_node(node_online_map);
-
-                /* To shrink on this node, there must be a surplus page */
-                if (delta < 0 && !h->surplus_huge_pages_node[nid])
-                        continue;
-                /* Surplus cannot exceed the total number of pages */
-                if (delta > 0 && h->surplus_huge_pages_node[nid] >=
-                                                h->nr_huge_pages_node[nid])
-                        continue;
-
-                h->surplus_huge_pages += delta;
-                h->surplus_huge_pages_node[nid] += delta;
-                ret = 1;
-                break;
-        } while (nid != prev_nid);
-
-        prev_nid = nid;
-        return ret;
-}
-
 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
 {
         set_compound_page_dtor(page, free_huge_page);
@@ -623,6 +588,34 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
         put_page(page); /* free it into the hugepage allocator */
 }
 
+static void prep_compound_gigantic_page(struct page *page, unsigned long order)
+{
+        int i;
+        int nr_pages = 1 << order;
+        struct page *p = page + 1;
+
+        /* we rely on prep_new_huge_page to set the destructor */
+        set_compound_order(page, order);
+        __SetPageHead(page);
+        for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
+                __SetPageTail(p);
+                p->first_page = page;
+        }
+}
+
+int PageHuge(struct page *page)
+{
+        compound_page_dtor *dtor;
+
+        if (!PageCompound(page))
+                return 0;
+
+        page = compound_head(page);
+        dtor = get_compound_page_dtor(page);
+
+        return dtor == free_huge_page;
+}
+
 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
 {
         struct page *page;
@@ -630,7 +623,7 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
         if (h->order >= MAX_ORDER)
                 return NULL;
 
-        page = alloc_pages_node(nid,
+        page = alloc_pages_exact_node(nid,
                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
                                                 __GFP_REPEAT|__GFP_NOWARN,
                 huge_page_order(h));
@@ -649,7 +642,7 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
  * Use a helper variable to find the next node and then
  * copy it back to hugetlb_next_nid afterwards:
  * otherwise there's a window in which a racer might
- * pass invalid nid MAX_NUMNODES to alloc_pages_node.
+ * pass invalid nid MAX_NUMNODES to alloc_pages_exact_node.
  * But we don't need to use a spin_lock here: it really
  * doesn't matter if occasionally a racer chooses the
  * same nid as we do.  Move nid forward in the mask even
@@ -875,7 +868,7 @@ static void return_unused_surplus_pages(struct hstate *h,
          * can no longer free unreserved surplus pages. This occurs when
          * the nodes with surplus pages have no free pages.
          */
-        unsigned long remaining_iterations = num_online_nodes();
+        unsigned long remaining_iterations = nr_online_nodes;
 
         /* Uncommit the reservation */
         h->resv_huge_pages -= unused_resv_pages;
@@ -904,7 +897,7 @@ static void return_unused_surplus_pages(struct hstate *h,
                         h->surplus_huge_pages--;
                         h->surplus_huge_pages_node[nid]--;
                         nr_pages--;
-                        remaining_iterations = num_online_nodes();
+                        remaining_iterations = nr_online_nodes;
                 }
         }
 }
@@ -1140,6 +1133,41 @@ static inline void try_to_free_low(struct hstate *h, unsigned long count)
 }
 #endif
 
+/*
+ * Increment or decrement surplus_huge_pages.  Keep node-specific counters
+ * balanced by operating on them in a round-robin fashion.
+ * Returns 1 if an adjustment was made.
+ */
+static int adjust_pool_surplus(struct hstate *h, int delta)
+{
+        static int prev_nid;
+        int nid = prev_nid;
+        int ret = 0;
+
+        VM_BUG_ON(delta != -1 && delta != 1);
+        do {
+                nid = next_node(nid, node_online_map);
+                if (nid == MAX_NUMNODES)
+                        nid = first_node(node_online_map);
+
+                /* To shrink on this node, there must be a surplus page */
+                if (delta < 0 && !h->surplus_huge_pages_node[nid])
+                        continue;
+                /* Surplus cannot exceed the total number of pages */
+                if (delta > 0 && h->surplus_huge_pages_node[nid] >=
+                                                h->nr_huge_pages_node[nid])
+                        continue;
+
+                h->surplus_huge_pages += delta;
+                h->surplus_huge_pages_node[nid] += delta;
+                ret = 1;
+                break;
+        } while (nid != prev_nid);
+
+        prev_nid = nid;
+        return ret;
+}
+
 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count)
 {