1 files changed, 410 insertions, 87 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a712fb9e04ce..4a4f9219683f 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -57,6 +57,7 @@
 #include <linux/ftrace_event.h>
 #include <linux/memcontrol.h>
 #include <linux/prefetch.h>
+#include <linux/migrate.h>
 #include <linux/page-debug-flags.h>
 
 #include <asm/tlbflush.h>
@@ -513,10 +514,10 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
  * free pages of length of (1 << order) and marked with _mapcount -2. Page's
  * order is recorded in page_private(page) field.
  * So when we are allocating or freeing one, we can derive the state of the
- * other.  That is, if we allocate a small block, and both were   
- * free, the remainder of the region must be split into blocks.   
+ * other.  That is, if we allocate a small block, and both were
+ * free, the remainder of the region must be split into blocks.
  * If a block is freed, and its buddy is also free, then this
- * triggers coalescing into a block of larger size.            
+ * triggers coalescing into a block of larger size.
  *
  * -- wli
  */
@@ -749,6 +750,24 @@ void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
         __free_pages(page, order);
 }
 
+#ifdef CONFIG_CMA
+/* Free whole pageblock and set it's migration type to MIGRATE_CMA. */
+void __init init_cma_reserved_pageblock(struct page *page)
+{
+        unsigned i = pageblock_nr_pages;
+        struct page *p = page;
+
+        do {
+                __ClearPageReserved(p);
+                set_page_count(p, 0);
+        } while (++p, --i);
+
+        set_page_refcounted(page);
+        set_pageblock_migratetype(page, MIGRATE_CMA);
+        __free_pages(page, pageblock_order);
+        totalram_pages += pageblock_nr_pages;
+}
+#endif
 
 /*
  * The order of subdivision here is critical for the IO subsystem.
@@ -874,11 +893,17 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  * This array describes the order lists are fallen back to when
  * the free lists for the desirable migrate type are depleted
  */
-static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
-        [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_RESERVE },
-        [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_RESERVE },
-        [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
-        [MIGRATE_RESERVE]     = { MIGRATE_RESERVE,     MIGRATE_RESERVE,   MIGRATE_RESERVE }, /* Never used */
+static int fallbacks[MIGRATE_TYPES][4] = {
+        [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,     MIGRATE_RESERVE },
+        [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,     MIGRATE_RESERVE },
+#ifdef CONFIG_CMA
+        [MIGRATE_MOVABLE]     = { MIGRATE_CMA,         MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
+        [MIGRATE_CMA]         = { MIGRATE_RESERVE }, /* Never used */
+#else
+        [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE,   MIGRATE_RESERVE },
+#endif
+        [MIGRATE_RESERVE]     = { MIGRATE_RESERVE }, /* Never used */
+        [MIGRATE_ISOLATE]     = { MIGRATE_RESERVE }, /* Never used */
 };
 
 /*
@@ -973,12 +998,12 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
         /* Find the largest possible block of pages in the other list */
         for (current_order = MAX_ORDER-1; current_order >= order;
                                                 --current_order) {
-                for (i = 0; i < MIGRATE_TYPES - 1; i++) {
+                for (i = 0;; i++) {
                         migratetype = fallbacks[start_migratetype][i];
 
                         /* MIGRATE_RESERVE handled later if necessary */
                         if (migratetype == MIGRATE_RESERVE)
-                                continue;
+                                break;
 
                         area = &(zone->free_area[current_order]);
                         if (list_empty(&area->free_list[migratetype]))
@@ -993,11 +1018,18 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
                          * pages to the preferred allocation list. If falling
                          * back for a reclaimable kernel allocation, be more
                          * aggressive about taking ownership of free pages
+                         *
+                         * On the other hand, never change migration
+                         * type of MIGRATE_CMA pageblocks nor move CMA
+                         * pages on different free lists. We don't
+                         * want unmovable pages to be allocated from
+                         * MIGRATE_CMA areas.
                          */
-                        if (unlikely(current_order >= (pageblock_order >> 1)) ||
-                                        start_migratetype == MIGRATE_RECLAIMABLE ||
-                                        page_group_by_mobility_disabled) {
-                                unsigned long pages;
+                        if (!is_migrate_cma(migratetype) &&
+                            (unlikely(current_order >= pageblock_order / 2) ||
+                             start_migratetype == MIGRATE_RECLAIMABLE ||
+                             page_group_by_mobility_disabled)) {
+                                int pages;
                                 pages = move_freepages_block(zone, page,
                                                                 start_migratetype);
 
@@ -1015,11 +1047,14 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
                         rmv_page_order(page);
 
                         /* Take ownership for orders >= pageblock_order */
-                        if (current_order >= pageblock_order)
+                        if (current_order >= pageblock_order &&
+                            !is_migrate_cma(migratetype))
                                 change_pageblock_range(page, current_order,
                                                         start_migratetype);
 
-                        expand(zone, page, order, current_order, area, migratetype);
+                        expand(zone, page, order, current_order, area,
+                               is_migrate_cma(migratetype)
+                             ? migratetype : start_migratetype);
 
                         trace_mm_page_alloc_extfrag(page, order, current_order,
                                 start_migratetype, migratetype);
@@ -1061,17 +1096,17 @@ retry_reserve:
         return page;
 }
 
-/* 
+/*
  * Obtain a specified number of elements from the buddy allocator, all under
  * a single hold of the lock, for efficiency.  Add them to the supplied list.
  * Returns the number of new pages which were placed at *list.
  */
-static int rmqueue_bulk(struct zone *zone, unsigned int order, 
+static int rmqueue_bulk(struct zone *zone, unsigned int order,
                         unsigned long count, struct list_head *list,
                         int migratetype, int cold)
 {
-        int i;
-        
+        int mt = migratetype, i;
+
         spin_lock(&zone->lock);
         for (i = 0; i < count; ++i) {
                 struct page *page = __rmqueue(zone, order, migratetype);
@@ -1091,7 +1126,12 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                         list_add(&page->lru, list);
                 else
                         list_add_tail(&page->lru, list);
-                set_page_private(page, migratetype);
+                if (IS_ENABLED(CONFIG_CMA)) {
+                        mt = get_pageblock_migratetype(page);
+                        if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE)
+                                mt = migratetype;
+                }
+                set_page_private(page, mt);
                 list = &page->lru;
         }
         __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
@@ -1371,8 +1411,12 @@ int split_free_page(struct page *page)
 
         if (order >= pageblock_order - 1) {
                 struct page *endpage = page + (1 << order) - 1;
-                for (; page < endpage; page += pageblock_nr_pages)
-                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+                for (; page < endpage; page += pageblock_nr_pages) {
+                        int mt = get_pageblock_migratetype(page);
+                        if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt))
+                                set_pageblock_migratetype(page,
+                                                          MIGRATE_MOVABLE);
+                }
         }
 
         return 1 << order;
@@ -2086,16 +2130,13 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 }
 #endif /* CONFIG_COMPACTION */
 
-/* The really slow allocator path where we enter direct reclaim */
-static inline struct page *
-__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
-        struct zonelist *zonelist, enum zone_type high_zoneidx,
-        nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
-        int migratetype, unsigned long *did_some_progress)
+/* Perform direct synchronous page reclaim */
+static int
+__perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
+                  nodemask_t *nodemask)
 {
-        struct page *page = NULL;
         struct reclaim_state reclaim_state;
-        bool drained = false;
+        int progress;
 
         cond_resched();
 
@@ -2106,7 +2147,7 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
         reclaim_state.reclaimed_slab = 0;
         current->reclaim_state = &reclaim_state;
 
-        *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
+        progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
 
         current->reclaim_state = NULL;
         lockdep_clear_current_reclaim_state();
@@ -2114,6 +2155,21 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
 
         cond_resched();
 
+        return progress;
+}
+
+/* The really slow allocator path where we enter direct reclaim */
+static inline struct page *
+__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
+        int migratetype, unsigned long *did_some_progress)
+{
+        struct page *page = NULL;
+        bool drained = false;
+
+        *did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
+                                               nodemask);
         if (unlikely(!(*did_some_progress)))
                 return NULL;
 
@@ -4244,25 +4300,24 @@ static inline void setup_usemap(struct pglist_data *pgdat,
 
 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 
-/* Return a sensible default order for the pageblock size. */
-static inline int pageblock_default_order(void)
-{
-        if (HPAGE_SHIFT > PAGE_SHIFT)
-                return HUGETLB_PAGE_ORDER;
-
-        return MAX_ORDER-1;
-}
-
 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
-static inline void __init set_pageblock_order(unsigned int order)
+static inline void __init set_pageblock_order(void)
 {
+        unsigned int order;
+
         /* Check that pageblock_nr_pages has not already been setup */
         if (pageblock_order)
                 return;
 
+        if (HPAGE_SHIFT > PAGE_SHIFT)
+                order = HUGETLB_PAGE_ORDER;
+        else
+                order = MAX_ORDER - 1;
+
         /*
          * Assume the largest contiguous order of interest is a huge page.
-         * This value may be variable depending on boot parameters on IA64
+         * This value may be variable depending on boot parameters on IA64 and
+         * powerpc.
          */
         pageblock_order = order;
 }
@@ -4270,15 +4325,13 @@ static inline void __init set_pageblock_order(unsigned int order)
 
 /*
  * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
- * and pageblock_default_order() are unused as pageblock_order is set
- * at compile-time. See include/linux/pageblock-flags.h for the values of
- * pageblock_order based on the kernel config
+ * is unused as pageblock_order is set at compile-time. See
+ * include/linux/pageblock-flags.h for the values of pageblock_order based on
+ * the kernel config
  */
-static inline int pageblock_default_order(unsigned int order)
+static inline void set_pageblock_order(void)
 {
-        return MAX_ORDER-1;
 }
-#define set_pageblock_order(x)  do {} while (0)
 
 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
 
@@ -4301,11 +4354,10 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
         init_waitqueue_head(&pgdat->kswapd_wait);
         pgdat->kswapd_max_order = 0;
         pgdat_page_cgroup_init(pgdat);
-        
+
         for (j = 0; j < MAX_NR_ZONES; j++) {
                 struct zone *zone = pgdat->node_zones + j;
                 unsigned long size, realsize, memmap_pages;
-                enum lru_list lru;
 
                 size = zone_spanned_pages_in_node(nid, j, zones_size);
                 realsize = size - zone_absent_pages_in_node(nid, j,
@@ -4355,18 +4407,13 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                 zone->zone_pgdat = pgdat;
 
                 zone_pcp_init(zone);
-                for_each_lru(lru)
-                        INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
-                zone->reclaim_stat.recent_rotated[0] = 0;
-                zone->reclaim_stat.recent_rotated[1] = 0;
-                zone->reclaim_stat.recent_scanned[0] = 0;
-                zone->reclaim_stat.recent_scanned[1] = 0;
+                lruvec_init(&zone->lruvec, zone);
                 zap_zone_vm_stats(zone);
                 zone->flags = 0;
                 if (!size)
                         continue;
 
-                set_pageblock_order(pageblock_default_order());
+                set_pageblock_order();
                 setup_usemap(pgdat, zone, size);
                 ret = init_currently_empty_zone(zone, zone_start_pfn,
                                                 size, MEMMAP_EARLY);
@@ -4759,31 +4806,34 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
         find_zone_movable_pfns_for_nodes();
 
         /* Print out the zone ranges */
-        printk("Zone PFN ranges:\n");
+        printk("Zone ranges:\n");
         for (i = 0; i < MAX_NR_ZONES; i++) {
                 if (i == ZONE_MOVABLE)
                         continue;
-                printk("  %-8s ", zone_names[i]);
+                printk(KERN_CONT "  %-8s ", zone_names[i]);
                 if (arch_zone_lowest_possible_pfn[i] ==
                                 arch_zone_highest_possible_pfn[i])
-                        printk("empty\n");
+                        printk(KERN_CONT "empty\n");
                 else
-                        printk("%0#10lx -> %0#10lx\n",
-                                arch_zone_lowest_possible_pfn[i],
-                                arch_zone_highest_possible_pfn[i]);
+                        printk(KERN_CONT "[mem %0#10lx-%0#10lx]\n",
+                                arch_zone_lowest_possible_pfn[i] << PAGE_SHIFT,
+                                (arch_zone_highest_possible_pfn[i]
+                                        << PAGE_SHIFT) - 1);
         }
 
         /* Print out the PFNs ZONE_MOVABLE begins at in each node */
-        printk("Movable zone start PFN for each node\n");
+        printk("Movable zone start for each node\n");
         for (i = 0; i < MAX_NUMNODES; i++) {
                 if (zone_movable_pfn[i])
-                        printk("  Node %d: %lu\n", i, zone_movable_pfn[i]);
+                        printk("  Node %d: %#010lx\n", i,
+                               zone_movable_pfn[i] << PAGE_SHIFT);
         }
 
         /* Print out the early_node_map[] */
-        printk("Early memory PFN ranges\n");
+        printk("Early memory node ranges\n");
         for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
-                printk("  %3d: %0#10lx -> %0#10lx\n", nid, start_pfn, end_pfn);
+                printk("  node %3d: [mem %#010lx-%#010lx]\n", nid,
+                       start_pfn << PAGE_SHIFT, (end_pfn << PAGE_SHIFT) - 1);
 
         /* Initialise every node */
         mminit_verify_pageflags_layout();
@@ -4976,14 +5026,7 @@ static void setup_per_zone_lowmem_reserve(void)
         calculate_totalreserve_pages();
 }
 
-/**
- * setup_per_zone_wmarks - called when min_free_kbytes changes
- * or when memory is hot-{added|removed}
- *
- * Ensures that the watermark[min,low,high] values for each zone are set
- * correctly with respect to min_free_kbytes.
- */
-void setup_per_zone_wmarks(void)
+static void __setup_per_zone_wmarks(void)
 {
         unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
         unsigned long lowmem_pages = 0;
@@ -5030,6 +5073,11 @@ void setup_per_zone_wmarks(void)
 
                 zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + (tmp >> 2);
                 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
+
+                zone->watermark[WMARK_MIN] += cma_wmark_pages(zone);
+                zone->watermark[WMARK_LOW] += cma_wmark_pages(zone);
+                zone->watermark[WMARK_HIGH] += cma_wmark_pages(zone);
+
                 setup_zone_migrate_reserve(zone);
                 spin_unlock_irqrestore(&zone->lock, flags);
         }
@@ -5038,6 +5086,20 @@ void setup_per_zone_wmarks(void)
         calculate_totalreserve_pages();
 }
 
+/**
+ * setup_per_zone_wmarks - called when min_free_kbytes changes
+ * or when memory is hot-{added|removed}
+ *
+ * Ensures that the watermark[min,low,high] values for each zone are set
+ * correctly with respect to min_free_kbytes.
+ */
+void setup_per_zone_wmarks(void)
+{
+        mutex_lock(&zonelists_mutex);
+        __setup_per_zone_wmarks();
+        mutex_unlock(&zonelists_mutex);
+}
+
 /*
  * The inactive anon list should be small enough that the VM never has to
  * do too much work, but large enough that each inactive page has a chance
@@ -5203,7 +5265,7 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
         int ret;
 
         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
-        if (!write || (ret == -EINVAL))
+        if (!write || (ret < 0))
                 return ret;
         for_each_populated_zone(zone) {
                 for_each_possible_cpu(cpu) {
@@ -5242,9 +5304,10 @@ void *__init alloc_large_system_hash(const char *tablename,
                                      int flags,
                                      unsigned int *_hash_shift,
                                      unsigned int *_hash_mask,
-                                     unsigned long limit)
+                                     unsigned long low_limit,
+                                     unsigned long high_limit)
 {
-        unsigned long long max = limit;
+        unsigned long long max = high_limit;
         unsigned long log2qty, size;
         void *table = NULL;
 
@@ -5282,6 +5345,8 @@ void *__init alloc_large_system_hash(const char *tablename,
         }
         max = min(max, 0x80000000ULL);
 
+        if (numentries < low_limit)
+                numentries = low_limit;
         if (numentries > max)
                 numentries = max;
 
@@ -5412,14 +5477,16 @@ static int
 __count_immobile_pages(struct zone *zone, struct page *page, int count)
 {
         unsigned long pfn, iter, found;
+        int mt;
+
         /*
          * For avoiding noise data, lru_add_drain_all() should be called
          * If ZONE_MOVABLE, the zone never contains immobile pages
          */
         if (zone_idx(zone) == ZONE_MOVABLE)
                 return true;
-
-        if (get_pageblock_migratetype(page) == MIGRATE_MOVABLE)
+        mt = get_pageblock_migratetype(page);
+        if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
                 return true;
 
         pfn = page_to_pfn(page);
@@ -5536,7 +5603,7 @@ out:
         return ret;
 }
 
-void unset_migratetype_isolate(struct page *page)
+void unset_migratetype_isolate(struct page *page, unsigned migratetype)
 {
         struct zone *zone;
         unsigned long flags;
@@ -5544,12 +5611,264 @@ void unset_migratetype_isolate(struct page *page)
         spin_lock_irqsave(&zone->lock, flags);
         if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
                 goto out;
-        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
-        move_freepages_block(zone, page, MIGRATE_MOVABLE);
+        set_pageblock_migratetype(page, migratetype);
+        move_freepages_block(zone, page, migratetype);
 out:
         spin_unlock_irqrestore(&zone->lock, flags);
 }
 
+#ifdef CONFIG_CMA
+
+static unsigned long pfn_max_align_down(unsigned long pfn)
+{
+        return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
+                             pageblock_nr_pages) - 1);
+}
+
+static unsigned long pfn_max_align_up(unsigned long pfn)
+{
+        return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
+                                pageblock_nr_pages));
+}
+
+static struct page *
+__alloc_contig_migrate_alloc(struct page *page, unsigned long private,
+                             int **resultp)
+{
+        gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE;
+
+        if (PageHighMem(page))
+                gfp_mask |= __GFP_HIGHMEM;
+
+        return alloc_page(gfp_mask);
+}
+
+/* [start, end) must belong to a single zone. */
+static int __alloc_contig_migrate_range(unsigned long start, unsigned long end)
+{
+        /* This function is based on compact_zone() from compaction.c. */
+
+        unsigned long pfn = start;
+        unsigned int tries = 0;
+        int ret = 0;
+
+        struct compact_control cc = {
+                .nr_migratepages = 0,
+                .order = -1,
+                .zone = page_zone(pfn_to_page(start)),
+                .sync = true,
+        };
+        INIT_LIST_HEAD(&cc.migratepages);
+
+        migrate_prep_local();
+
+        while (pfn < end || !list_empty(&cc.migratepages)) {
+                if (fatal_signal_pending(current)) {
+                        ret = -EINTR;
+                        break;
+                }
+
+                if (list_empty(&cc.migratepages)) {
+                        cc.nr_migratepages = 0;
+                        pfn = isolate_migratepages_range(cc.zone, &cc,
+                                                         pfn, end);
+                        if (!pfn) {
+                                ret = -EINTR;
+                                break;
+                        }
+                        tries = 0;
+                } else if (++tries == 5) {
+                        ret = ret < 0 ? ret : -EBUSY;
+                        break;
+                }
+
+                ret = migrate_pages(&cc.migratepages,
+                                    __alloc_contig_migrate_alloc,
+                                    0, false, MIGRATE_SYNC);
+        }
+
+        putback_lru_pages(&cc.migratepages);
+        return ret > 0 ? 0 : ret;
+}
+
+/*
+ * Update zone's cma pages counter used for watermark level calculation.
+ */
+static inline void __update_cma_watermarks(struct zone *zone, int count)
+{
+        unsigned long flags;
+        spin_lock_irqsave(&zone->lock, flags);
+        zone->min_cma_pages += count;
+        spin_unlock_irqrestore(&zone->lock, flags);
+        setup_per_zone_wmarks();
+}
+
+/*
+ * Trigger memory pressure bump to reclaim some pages in order to be able to
+ * allocate 'count' pages in single page units. Does similar work as
+ *__alloc_pages_slowpath() function.
+ */
+static int __reclaim_pages(struct zone *zone, gfp_t gfp_mask, int count)
+{
+        enum zone_type high_zoneidx = gfp_zone(gfp_mask);
+        struct zonelist *zonelist = node_zonelist(0, gfp_mask);
+        int did_some_progress = 0;
+        int order = 1;
+
+        /*
+         * Increase level of watermarks to force kswapd do his job
+         * to stabilise at new watermark level.
+         */
+        __update_cma_watermarks(zone, count);
+
+        /* Obey watermarks as if the page was being allocated */
+        while (!zone_watermark_ok(zone, 0, low_wmark_pages(zone), 0, 0)) {
+                wake_all_kswapd(order, zonelist, high_zoneidx, zone_idx(zone));
+
+                did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
+                                                      NULL);
+                if (!did_some_progress) {
+                        /* Exhausted what can be done so it's blamo time */
+                        out_of_memory(zonelist, gfp_mask, order, NULL, false);
+                }
+        }
+
+        /* Restore original watermark levels. */
+        __update_cma_watermarks(zone, -count);
+
+        return count;
+}
+
+/**
+ * alloc_contig_range() -- tries to allocate given range of pages
+ * @start:      start PFN to allocate
+ * @end:        one-past-the-last PFN to allocate
+ * @migratetype:        migratetype of the underlaying pageblocks (either
+ *                      #MIGRATE_MOVABLE or #MIGRATE_CMA).  All pageblocks
+ *                      in range must have the same migratetype and it must
+ *                      be either of the two.
+ *
+ * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
+ * aligned, however it's the caller's responsibility to guarantee that
+ * we are the only thread that changes migrate type of pageblocks the
+ * pages fall in.
+ *
+ * The PFN range must belong to a single zone.
+ *
+ * Returns zero on success or negative error code.  On success all
+ * pages which PFN is in [start, end) are allocated for the caller and
+ * need to be freed with free_contig_range().
+ */
+int alloc_contig_range(unsigned long start, unsigned long end,
+                       unsigned migratetype)
+{
+        struct zone *zone = page_zone(pfn_to_page(start));
+        unsigned long outer_start, outer_end;
+        int ret = 0, order;
+
+        /*
+         * What we do here is we mark all pageblocks in range as
+         * MIGRATE_ISOLATE.  Because pageblock and max order pages may
+         * have different sizes, and due to the way page allocator
+         * work, we align the range to biggest of the two pages so
+         * that page allocator won't try to merge buddies from
+         * different pageblocks and change MIGRATE_ISOLATE to some
+         * other migration type.
+         *
+         * Once the pageblocks are marked as MIGRATE_ISOLATE, we
+         * migrate the pages from an unaligned range (ie. pages that
+         * we are interested in).  This will put all the pages in
+         * range back to page allocator as MIGRATE_ISOLATE.
+         *
+         * When this is done, we take the pages in range from page
+         * allocator removing them from the buddy system.  This way
+         * page allocator will never consider using them.
+         *
+         * This lets us mark the pageblocks back as
+         * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
+         * aligned range but not in the unaligned, original range are
+         * put back to page allocator so that buddy can use them.
+         */
+
+        ret = start_isolate_page_range(pfn_max_align_down(start),
+                                       pfn_max_align_up(end), migratetype);
+        if (ret)
+                goto done;
+
+        ret = __alloc_contig_migrate_range(start, end);
+        if (ret)
+                goto done;
+
+        /*
+         * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
+         * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
+         * more, all pages in [start, end) are free in page allocator.
+         * What we are going to do is to allocate all pages from
+         * [start, end) (that is remove them from page allocator).
+         *
+         * The only problem is that pages at the beginning and at the
+         * end of interesting range may be not aligned with pages that
+         * page allocator holds, ie. they can be part of higher order
+         * pages.  Because of this, we reserve the bigger range and
+         * once this is done free the pages we are not interested in.
+         *
+         * We don't have to hold zone->lock here because the pages are
+         * isolated thus they won't get removed from buddy.
+         */
+
+        lru_add_drain_all();
+        drain_all_pages();
+
+        order = 0;
+        outer_start = start;
+        while (!PageBuddy(pfn_to_page(outer_start))) {
+                if (++order >= MAX_ORDER) {
+                        ret = -EBUSY;
+                        goto done;
+                }
+                outer_start &= ~0UL << order;
+        }
+
+        /* Make sure the range is really isolated. */
+        if (test_pages_isolated(outer_start, end)) {
+                pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n",
+                       outer_start, end);
+                ret = -EBUSY;
+                goto done;
+        }
+
+        /*
+         * Reclaim enough pages to make sure that contiguous allocation
+         * will not starve the system.
+         */
+        __reclaim_pages(zone, GFP_HIGHUSER_MOVABLE, end-start);
+
+        /* Grab isolated pages from freelists. */
+        outer_end = isolate_freepages_range(outer_start, end);
+        if (!outer_end) {
+                ret = -EBUSY;
+                goto done;
+        }
+
+        /* Free head and tail (if any) */
+        if (start != outer_start)
+                free_contig_range(outer_start, start - outer_start);
+        if (end != outer_end)
+                free_contig_range(end, outer_end - end);
+
+done:
+        undo_isolate_page_range(pfn_max_align_down(start),
+                                pfn_max_align_up(end), migratetype);
+        return ret;
+}
+
+void free_contig_range(unsigned long pfn, unsigned nr_pages)
+{
+        for (; nr_pages--; ++pfn)
+                __free_page(pfn_to_page(pfn));
+}
+#endif
+
 #ifdef CONFIG_MEMORY_HOTREMOVE
 /*
  * All pages in the range must be isolated before calling this.
@@ -5618,7 +5937,7 @@ bool is_free_buddy_page(struct page *page)
 }
 #endif
 
-static struct trace_print_flags pageflag_names[] = {
+static const struct trace_print_flags pageflag_names[] = {
         {1UL << PG_locked,              "locked"        },
         {1UL << PG_error,               "error"         },
         {1UL << PG_referenced,          "referenced"    },
@@ -5653,7 +5972,9 @@ static struct trace_print_flags pageflag_names[] = {
 #ifdef CONFIG_MEMORY_FAILURE
         {1UL << PG_hwpoison,            "hwpoison"      },
 #endif
-        {-1UL,                          NULL            },
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+        {1UL << PG_compound_lock,       "compound_lock" },
+#endif
 };
 
 static void dump_page_flags(unsigned long flags)
@@ -5662,12 +5983,14 @@ static void dump_page_flags(unsigned long flags)
         unsigned long mask;
         int i;
 
+        BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS);
+
         printk(KERN_ALERT "page flags: %#lx(", flags);
 
         /* remove zone id */
         flags &= (1UL << NR_PAGEFLAGS) - 1;
 
-        for (i = 0; pageflag_names[i].name && flags; i++) {
+        for (i = 0; i < ARRAY_SIZE(pageflag_names) && flags; i++) {
 
                 mask = pageflag_names[i].mask;
                 if ((flags & mask) != mask)