hugetlb: Try to grow hugetlb pool for MAP_PRIVATE mappings

Because we overcommit hugepages for MAP_PRIVATE mappings, it is possible that
the hugetlb pool will be exhausted or completely reserved when a hugepage is
needed to satisfy a page fault.  Before killing the process in this situation,
try to allocate a hugepage directly from the buddy allocator.

The explicitly configured pool size becomes a low watermark.  When dynamically
grown, the allocated huge pages are accounted as a surplus over the watermark.
 As huge pages are freed on a node, surplus pages are released to the buddy
allocator so that the pool will shrink back to the watermark.

Surplus accounting also allows for friendlier explicit pool resizing.  When
shrinking a pool that is fully in-use, increase the surplus so pages will be
returned to the buddy allocator as soon as they are freed.  When growing a
pool that has a surplus, consume the surplus first and then allocate new
pages.

Signed-off-by: Adam Litke <agl@us.ibm.com>
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Acked-by: Dave McCracken <dave.mccracken@oracle.com>
Cc: William Irwin <bill.irwin@oracle.com>
Cc: David Gibson <david@gibson.dropbear.id.au>
Cc: Ken Chen <kenchen@google.com>
Cc: Badari Pulavarty <pbadari@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 125 insertions, 14 deletions

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ba029d640740..8768e5250323 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -23,10 +23,12 @@
 
 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
 static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
+static unsigned long surplus_huge_pages;
 unsigned long max_huge_pages;
 static struct list_head hugepage_freelists[MAX_NUMNODES];
 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
 static unsigned int free_huge_pages_node[MAX_NUMNODES];
+static unsigned int surplus_huge_pages_node[MAX_NUMNODES];
 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
 unsigned long hugepages_treat_as_movable;
 
@@ -109,15 +111,57 @@ static void update_and_free_page(struct page *page)
 
 static void free_huge_page(struct page *page)
 {
-        BUG_ON(page_count(page));
+        int nid = page_to_nid(page);
 
+        BUG_ON(page_count(page));
         INIT_LIST_HEAD(&page->lru);
 
         spin_lock(&hugetlb_lock);
-        enqueue_huge_page(page);
+        if (surplus_huge_pages_node[nid]) {
+                update_and_free_page(page);
+                surplus_huge_pages--;
+                surplus_huge_pages_node[nid]--;
+        } else {
+                enqueue_huge_page(page);
+        }
         spin_unlock(&hugetlb_lock);
 }
 
+/*
+ * Increment or decrement surplus_huge_pages.  Keep node-specific counters
+ * balanced by operating on them in a round-robin fashion.
+ * Returns 1 if an adjustment was made.
+ */
+static int adjust_pool_surplus(int delta)
+{
+        static int prev_nid;
+        int nid = prev_nid;
+        int ret = 0;
+
+        VM_BUG_ON(delta != -1 && delta != 1);
+        do {
+                nid = next_node(nid, node_online_map);
+                if (nid == MAX_NUMNODES)
+                        nid = first_node(node_online_map);
+
+                /* To shrink on this node, there must be a surplus page */
+                if (delta < 0 && !surplus_huge_pages_node[nid])
+                        continue;
+                /* Surplus cannot exceed the total number of pages */
+                if (delta > 0 && surplus_huge_pages_node[nid] >=
+                                                nr_huge_pages_node[nid])
+                        continue;
+
+                surplus_huge_pages += delta;
+                surplus_huge_pages_node[nid] += delta;
+                ret = 1;
+                break;
+        } while (nid != prev_nid);
+
+        prev_nid = nid;
+        return ret;
+}
+
 static int alloc_fresh_huge_page(void)
 {
         static int prev_nid;
@@ -150,10 +194,30 @@ static int alloc_fresh_huge_page(void)
         return 0;
 }
 
+static struct page *alloc_buddy_huge_page(struct vm_area_struct *vma,
+                                                unsigned long address)
+{
+        struct page *page;
+
+        page = alloc_pages(htlb_alloc_mask|__GFP_COMP|__GFP_NOWARN,
+                                        HUGETLB_PAGE_ORDER);
+        if (page) {
+                set_compound_page_dtor(page, free_huge_page);
+                spin_lock(&hugetlb_lock);
+                nr_huge_pages++;
+                nr_huge_pages_node[page_to_nid(page)]++;
+                surplus_huge_pages++;
+                surplus_huge_pages_node[page_to_nid(page)]++;
+                spin_unlock(&hugetlb_lock);
+        }
+
+        return page;
+}
+
 static struct page *alloc_huge_page(struct vm_area_struct *vma,
                                     unsigned long addr)
 {
-        struct page *page;
+        struct page *page = NULL;
 
         spin_lock(&hugetlb_lock);
         if (vma->vm_flags & VM_MAYSHARE)
@@ -173,7 +237,16 @@ fail:
         if (vma->vm_flags & VM_MAYSHARE)
                 resv_huge_pages++;
         spin_unlock(&hugetlb_lock);
-        return NULL;
+
+        /*
+         * Private mappings do not use reserved huge pages so the allocation
+         * may have failed due to an undersized hugetlb pool.  Try to grab a
+         * surplus huge page from the buddy allocator.
+         */
+        if (!(vma->vm_flags & VM_MAYSHARE))
+                page = alloc_buddy_huge_page(vma, addr);
+
+        return page;
 }
 
 static int __init hugetlb_init(void)
@@ -241,26 +314,62 @@ static inline void try_to_free_low(unsigned long count)
 }
 #endif
 
+#define persistent_huge_pages (nr_huge_pages - surplus_huge_pages)
 static unsigned long set_max_huge_pages(unsigned long count)
 {
-        while (count > nr_huge_pages) {
-                if (!alloc_fresh_huge_page())
-                        return nr_huge_pages;
-        }
-        if (count >= nr_huge_pages)
-                return nr_huge_pages;
+        unsigned long min_count, ret;
 
+        /*
+         * Increase the pool size
+         * First take pages out of surplus state.  Then make up the
+         * remaining difference by allocating fresh huge pages.
+         */
         spin_lock(&hugetlb_lock);
-        count = max(count, resv_huge_pages);
-        try_to_free_low(count);
-        while (count < nr_huge_pages) {
+        while (surplus_huge_pages && count > persistent_huge_pages) {
+                if (!adjust_pool_surplus(-1))
+                        break;
+        }
+
+        while (count > persistent_huge_pages) {
+                int ret;
+                /*
+                 * If this allocation races such that we no longer need the
+                 * page, free_huge_page will handle it by freeing the page
+                 * and reducing the surplus.
+                 */
+                spin_unlock(&hugetlb_lock);
+                ret = alloc_fresh_huge_page();
+                spin_lock(&hugetlb_lock);
+                if (!ret)
+                        goto out;
+
+        }
+        if (count >= persistent_huge_pages)
+                goto out;
+
+        /*
+         * Decrease the pool size
+         * First return free pages to the buddy allocator (being careful
+         * to keep enough around to satisfy reservations).  Then place
+         * pages into surplus state as needed so the pool will shrink
+         * to the desired size as pages become free.
+         */
+        min_count = max(count, resv_huge_pages);
+        try_to_free_low(min_count);
+        while (min_count < persistent_huge_pages) {
                 struct page *page = dequeue_huge_page(NULL, 0);
                 if (!page)
                         break;
                 update_and_free_page(page);
         }
+        while (count < persistent_huge_pages) {
+                if (!adjust_pool_surplus(1))
+                        break;
+        }
+out:
+        ret = persistent_huge_pages;
         spin_unlock(&hugetlb_lock);
-        return nr_huge_pages;
+        return ret;
 }
 
 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
@@ -292,10 +401,12 @@ int hugetlb_report_meminfo(char *buf)
                         "HugePages_Total: %5lu\n"
                         "HugePages_Free:  %5lu\n"
                         "HugePages_Rsvd:  %5lu\n"
+                        "HugePages_Surp:  %5lu\n"
                         "Hugepagesize:    %5lu kB\n",
                         nr_huge_pages,
                         free_huge_pages,
                         resv_huge_pages,
+                        surplus_huge_pages,
                         HPAGE_SIZE/1024);
 }