1 files changed, 555 insertions, 381 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 5c44ed49ca93..a0de15f46987 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -23,6 +23,7 @@
 #include <linux/bootmem.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/kmemcheck.h>
 #include <linux/module.h>
 #include <linux/suspend.h>
 #include <linux/pagevec.h>
@@ -46,6 +47,7 @@
 #include <linux/page-isolation.h>
 #include <linux/page_cgroup.h>
 #include <linux/debugobjects.h>
+#include <linux/kmemleak.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>
@@ -71,6 +73,7 @@ unsigned long totalram_pages __read_mostly;
 unsigned long totalreserve_pages __read_mostly;
 unsigned long highest_memmap_pfn __read_mostly;
 int percpu_pagelist_fraction;
+gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 
 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 int pageblock_order __read_mostly;
@@ -149,10 +152,6 @@ static unsigned long __meminitdata dma_reserve;
   static int __meminitdata nr_nodemap_entries;
   static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
   static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
-#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
-  static unsigned long __meminitdata node_boundary_start_pfn[MAX_NUMNODES];
-  static unsigned long __meminitdata node_boundary_end_pfn[MAX_NUMNODES];
-#endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */
   static unsigned long __initdata required_kernelcore;
   static unsigned long __initdata required_movablecore;
   static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
@@ -164,17 +163,25 @@ static unsigned long __meminitdata dma_reserve;
 
 #if MAX_NUMNODES > 1
 int nr_node_ids __read_mostly = MAX_NUMNODES;
+int nr_online_nodes __read_mostly = 1;
 EXPORT_SYMBOL(nr_node_ids);
+EXPORT_SYMBOL(nr_online_nodes);
 #endif
 
 int page_group_by_mobility_disabled __read_mostly;
 
 static void set_pageblock_migratetype(struct page *page, int migratetype)
 {
+
+        if (unlikely(page_group_by_mobility_disabled))
+                migratetype = MIGRATE_UNMOVABLE;
+
         set_pageblock_flags_group(page, (unsigned long)migratetype,
                                         PB_migrate, PB_migrate_end);
 }
 
+bool oom_killer_disabled __read_mostly;
+
 #ifdef CONFIG_DEBUG_VM
 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 {
@@ -297,23 +304,6 @@ void prep_compound_page(struct page *page, unsigned long order)
         }
 }
 
-#ifdef CONFIG_HUGETLBFS
-void prep_compound_gigantic_page(struct page *page, unsigned long order)
-{
-        int i;
-        int nr_pages = 1 << order;
-        struct page *p = page + 1;
-
-        set_compound_page_dtor(page, free_compound_page);
-        set_compound_order(page, order);
-        __SetPageHead(page);
-        for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
-                __SetPageTail(p);
-                p->first_page = page;
-        }
-}
-#endif
-
 static int destroy_compound_page(struct page *page, unsigned long order)
 {
         int i;
@@ -331,7 +321,7 @@ static int destroy_compound_page(struct page *page, unsigned long order)
         for (i = 1; i < nr_pages; i++) {
                 struct page *p = page + i;
 
-                if (unlikely(!PageTail(p) | (p->first_page != page))) {
+                if (unlikely(!PageTail(p) || (p->first_page != page))) {
                         bad_page(page);
                         bad++;
                 }
@@ -420,7 +410,7 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
                 return 0;
 
         if (PageBuddy(buddy) && page_order(buddy) == order) {
-                BUG_ON(page_count(buddy) != 0);
+                VM_BUG_ON(page_count(buddy) != 0);
                 return 1;
         }
         return 0;
@@ -451,22 +441,22 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
  */
 
 static inline void __free_one_page(struct page *page,
-                struct zone *zone, unsigned int order)
+                struct zone *zone, unsigned int order,
+                int migratetype)
 {
         unsigned long page_idx;
-        int order_size = 1 << order;
-        int migratetype = get_pageblock_migratetype(page);
 
         if (unlikely(PageCompound(page)))
                 if (unlikely(destroy_compound_page(page, order)))
                         return;
 
+        VM_BUG_ON(migratetype == -1);
+
         page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
 
-        VM_BUG_ON(page_idx & (order_size - 1));
+        VM_BUG_ON(page_idx & ((1 << order) - 1));
         VM_BUG_ON(bad_range(zone, page));
 
-        __mod_zone_page_state(zone, NR_FREE_PAGES, order_size);
         while (order < MAX_ORDER-1) {
                 unsigned long combined_idx;
                 struct page *buddy;
@@ -490,12 +480,26 @@ static inline void __free_one_page(struct page *page,
         zone->free_area[order].nr_free++;
 }
 
+#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
+/*
+ * free_page_mlock() -- clean up attempts to free and mlocked() page.
+ * Page should not be on lru, so no need to fix that up.
+ * free_pages_check() will verify...
+ */
+static inline void free_page_mlock(struct page *page)
+{
+        __dec_zone_page_state(page, NR_MLOCK);
+        __count_vm_event(UNEVICTABLE_MLOCKFREED);
+}
+#else
+static void free_page_mlock(struct page *page) { }
+#endif
+
 static inline int free_pages_check(struct page *page)
 {
-        free_page_mlock(page);
         if (unlikely(page_mapcount(page) |
                 (page->mapping != NULL)  |
-                (page_count(page) != 0)  |
+                (atomic_read(&page->_count) != 0) |
                 (page->flags & PAGE_FLAGS_CHECK_AT_FREE))) {
                 bad_page(page);
                 return 1;
@@ -522,6 +526,8 @@ static void free_pages_bulk(struct zone *zone, int count,
         spin_lock(&zone->lock);
         zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
         zone->pages_scanned = 0;
+
+        __mod_zone_page_state(zone, NR_FREE_PAGES, count << order);
         while (count--) {
                 struct page *page;
 
@@ -529,17 +535,20 @@ static void free_pages_bulk(struct zone *zone, int count,
                 page = list_entry(list->prev, struct page, lru);
                 /* have to delete it as __free_one_page list manipulates */
                 list_del(&page->lru);
-                __free_one_page(page, zone, order);
+                __free_one_page(page, zone, order, page_private(page));
         }
         spin_unlock(&zone->lock);
 }
 
-static void free_one_page(struct zone *zone, struct page *page, int order)
+static void free_one_page(struct zone *zone, struct page *page, int order,
+                                int migratetype)
 {
         spin_lock(&zone->lock);
         zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
         zone->pages_scanned = 0;
-        __free_one_page(page, zone, order);
+
+        __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
+        __free_one_page(page, zone, order, migratetype);
         spin_unlock(&zone->lock);
 }
 
@@ -548,6 +557,9 @@ static void __free_pages_ok(struct page *page, unsigned int order)
         unsigned long flags;
         int i;
         int bad = 0;
+        int wasMlocked = TestClearPageMlocked(page);
+
+        kmemcheck_free_shadow(page, order);
 
         for (i = 0 ; i < (1 << order) ; ++i)
                 bad += free_pages_check(page + i);
@@ -563,8 +575,11 @@ static void __free_pages_ok(struct page *page, unsigned int order)
         kernel_map_pages(page, 1 << order, 0);
 
         local_irq_save(flags);
+        if (unlikely(wasMlocked))
+                free_page_mlock(page);
         __count_vm_events(PGFREE, 1 << order);
-        free_one_page(page_zone(page), page, order);
+        free_one_page(page_zone(page), page, order,
+                                        get_pageblock_migratetype(page));
         local_irq_restore(flags);
 }
 
@@ -635,7 +650,7 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
 {
         if (unlikely(page_mapcount(page) |
                 (page->mapping != NULL)  |
-                (page_count(page) != 0)  |
+                (atomic_read(&page->_count) != 0)  |
                 (page->flags & PAGE_FLAGS_CHECK_AT_PREP))) {
                 bad_page(page);
                 return 1;
@@ -660,7 +675,8 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  * Go through the free lists for the given migratetype and remove
  * the smallest available page from the freelists
  */
-static struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
+static inline
+struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
                                                 int migratetype)
 {
         unsigned int current_order;
@@ -678,7 +694,6 @@ static struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
                 list_del(&page->lru);
                 rmv_page_order(page);
                 area->nr_free--;
-                __mod_zone_page_state(zone, NR_FREE_PAGES, - (1UL << order));
                 expand(zone, page, order, current_order, area, migratetype);
                 return page;
         }
@@ -769,8 +784,8 @@ static int move_freepages_block(struct zone *zone, struct page *page,
 }
 
 /* Remove an element from the buddy allocator from the fallback list */
-static struct page *__rmqueue_fallback(struct zone *zone, int order,
-                                                int start_migratetype)
+static inline struct page *
+__rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
 {
         struct free_area * area;
         int current_order;
@@ -802,13 +817,15 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
                          * agressive about taking ownership of free pages
                          */
                         if (unlikely(current_order >= (pageblock_order >> 1)) ||
-                                        start_migratetype == MIGRATE_RECLAIMABLE) {
+                                        start_migratetype == MIGRATE_RECLAIMABLE ||
+                                        page_group_by_mobility_disabled) {
                                 unsigned long pages;
                                 pages = move_freepages_block(zone, page,
                                                                 start_migratetype);
 
                                 /* Claim the whole block if over half of it is free */
-                                if (pages >= (1 << (pageblock_order-1)))
+                                if (pages >= (1 << (pageblock_order-1)) ||
+                                                page_group_by_mobility_disabled)
                                         set_pageblock_migratetype(page,
                                                                 start_migratetype);
 
@@ -818,8 +835,6 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
                         /* Remove the page from the freelists */
                         list_del(&page->lru);
                         rmv_page_order(page);
-                        __mod_zone_page_state(zone, NR_FREE_PAGES,
-                                                        -(1UL << order));
 
                         if (current_order == pageblock_order)
                                 set_pageblock_migratetype(page,
@@ -830,8 +845,7 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
                 }
         }
 
-        /* Use MIGRATE_RESERVE rather than fail an allocation */
-        return __rmqueue_smallest(zone, order, MIGRATE_RESERVE);
+        return NULL;
 }
 
 /*
@@ -843,11 +857,23 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order,
 {
         struct page *page;
 
+retry_reserve:
         page = __rmqueue_smallest(zone, order, migratetype);
 
-        if (unlikely(!page))
+        if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
                 page = __rmqueue_fallback(zone, order, migratetype);
 
+                /*
+                 * Use MIGRATE_RESERVE rather than fail an allocation. goto
+                 * is used because __rmqueue_smallest is an inline function
+                 * and we want just one call site
+                 */
+                if (!page) {
+                        migratetype = MIGRATE_RESERVE;
+                        goto retry_reserve;
+                }
+        }
+
         return page;
 }
 
@@ -858,7 +884,7 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order,
  */
 static int rmqueue_bulk(struct zone *zone, unsigned int order, 
                         unsigned long count, struct list_head *list,
-                        int migratetype)
+                        int migratetype, int cold)
 {
         int i;
         
@@ -877,10 +903,14 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                  * merge IO requests if the physical pages are ordered
                  * properly.
                  */
-                list_add(&page->lru, list);
+                if (likely(cold == 0))
+                        list_add(&page->lru, list);
+                else
+                        list_add_tail(&page->lru, list);
                 set_page_private(page, migratetype);
                 list = &page->lru;
         }
+        __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
         spin_unlock(&zone->lock);
         return i;
 }
@@ -922,13 +952,10 @@ static void drain_pages(unsigned int cpu)
         unsigned long flags;
         struct zone *zone;
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 struct per_cpu_pageset *pset;
                 struct per_cpu_pages *pcp;
 
-                if (!populated_zone(zone))
-                        continue;
-
                 pset = zone_pcp(zone, cpu);
 
                 pcp = &pset->pcp;
@@ -999,6 +1026,9 @@ static void free_hot_cold_page(struct page *page, int cold)
         struct zone *zone = page_zone(page);
         struct per_cpu_pages *pcp;
         unsigned long flags;
+        int wasMlocked = TestClearPageMlocked(page);
+
+        kmemcheck_free_shadow(page, 0);
 
         if (PageAnon(page))
                 page->mapping = NULL;
@@ -1013,13 +1043,16 @@ static void free_hot_cold_page(struct page *page, int cold)
         kernel_map_pages(page, 1, 0);
 
         pcp = &zone_pcp(zone, get_cpu())->pcp;
+        set_page_private(page, get_pageblock_migratetype(page));
         local_irq_save(flags);
+        if (unlikely(wasMlocked))
+                free_page_mlock(page);
         __count_vm_event(PGFREE);
+
         if (cold)
                 list_add_tail(&page->lru, &pcp->list);
         else
                 list_add(&page->lru, &pcp->list);
-        set_page_private(page, get_pageblock_migratetype(page));
         pcp->count++;
         if (pcp->count >= pcp->high) {
                 free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
@@ -1053,6 +1086,16 @@ void split_page(struct page *page, unsigned int order)
 
         VM_BUG_ON(PageCompound(page));
         VM_BUG_ON(!page_count(page));
+
+#ifdef CONFIG_KMEMCHECK
+        /*
+         * Split shadow pages too, because free(page[0]) would
+         * otherwise free the whole shadow.
+         */
+        if (kmemcheck_page_is_tracked(page))
+                split_page(virt_to_page(page[0].shadow), order);
+#endif
+
         for (i = 1; i < (1 << order); i++)
                 set_page_refcounted(page + i);
 }
@@ -1062,14 +1105,15 @@ void split_page(struct page *page, unsigned int order)
  * we cheat by calling it from here, in the order > 0 path.  Saves a branch
  * or two.
  */
-static struct page *buffered_rmqueue(struct zone *preferred_zone,
-                        struct zone *zone, int order, gfp_t gfp_flags)
+static inline
+struct page *buffered_rmqueue(struct zone *preferred_zone,
+                        struct zone *zone, int order, gfp_t gfp_flags,
+                        int migratetype)
 {
         unsigned long flags;
         struct page *page;
         int cold = !!(gfp_flags & __GFP_COLD);
         int cpu;
-        int migratetype = allocflags_to_migratetype(gfp_flags);
 
 again:
         cpu  = get_cpu();
@@ -1080,7 +1124,8 @@ again:
                 local_irq_save(flags);
                 if (!pcp->count) {
                         pcp->count = rmqueue_bulk(zone, 0,
-                                        pcp->batch, &pcp->list, migratetype);
+                                        pcp->batch, &pcp->list,
+                                        migratetype, cold);
                         if (unlikely(!pcp->count))
                                 goto failed;
                 }
@@ -1099,15 +1144,30 @@ again:
                 /* Allocate more to the pcp list if necessary */
                 if (unlikely(&page->lru == &pcp->list)) {
                         pcp->count += rmqueue_bulk(zone, 0,
-                                        pcp->batch, &pcp->list, migratetype);
+                                        pcp->batch, &pcp->list,
+                                        migratetype, cold);
                         page = list_entry(pcp->list.next, struct page, lru);
                 }
 
                 list_del(&page->lru);
                 pcp->count--;
         } else {
+                if (unlikely(gfp_flags & __GFP_NOFAIL)) {
+                        /*
+                         * __GFP_NOFAIL is not to be used in new code.
+                         *
+                         * All __GFP_NOFAIL callers should be fixed so that they
+                         * properly detect and handle allocation failures.
+                         *
+                         * We most definitely don't want callers attempting to
+                         * allocate greater than order-1 page units with
+                         * __GFP_NOFAIL.
+                         */
+                        WARN_ON_ONCE(order > 1);
+                }
                 spin_lock_irqsave(&zone->lock, flags);
                 page = __rmqueue(zone, order, migratetype);
+                __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
                 spin_unlock(&zone->lock);
                 if (!page)
                         goto failed;
@@ -1129,10 +1189,15 @@ failed:
         return NULL;
 }
 
-#define ALLOC_NO_WATERMARKS     0x01 /* don't check watermarks at all */
-#define ALLOC_WMARK_MIN         0x02 /* use pages_min watermark */
-#define ALLOC_WMARK_LOW         0x04 /* use pages_low watermark */
-#define ALLOC_WMARK_HIGH        0x08 /* use pages_high watermark */
+/* The ALLOC_WMARK bits are used as an index to zone->watermark */
+#define ALLOC_WMARK_MIN         WMARK_MIN
+#define ALLOC_WMARK_LOW         WMARK_LOW
+#define ALLOC_WMARK_HIGH        WMARK_HIGH
+#define ALLOC_NO_WATERMARKS     0x04 /* don't check watermarks at all */
+
+/* Mask to get the watermark bits */
+#define ALLOC_WMARK_MASK        (ALLOC_NO_WATERMARKS-1)
+
 #define ALLOC_HARDER            0x10 /* try to alloc harder */
 #define ALLOC_HIGH              0x20 /* __GFP_HIGH set */
 #define ALLOC_CPUSET            0x40 /* check for correct cpuset */
@@ -1390,23 +1455,18 @@ static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  */
 static struct page *
 get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
-                struct zonelist *zonelist, int high_zoneidx, int alloc_flags)
+                struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
+                struct zone *preferred_zone, int migratetype)
 {
         struct zoneref *z;
         struct page *page = NULL;
         int classzone_idx;
-        struct zone *zone, *preferred_zone;
+        struct zone *zone;
         nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
         int zlc_active = 0;             /* set if using zonelist_cache */
         int did_zlc_setup = 0;          /* just call zlc_setup() one time */
 
-        (void)first_zones_zonelist(zonelist, high_zoneidx, nodemask,
-                                                        &preferred_zone);
-        if (!preferred_zone)
-                return NULL;
-
         classzone_idx = zone_idx(preferred_zone);
-
 zonelist_scan:
         /*
          * Scan zonelist, looking for a zone with enough free.
@@ -1421,31 +1481,49 @@ zonelist_scan:
                         !cpuset_zone_allowed_softwall(zone, gfp_mask))
                                 goto try_next_zone;
 
+                BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
                 if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
                         unsigned long mark;
-                        if (alloc_flags & ALLOC_WMARK_MIN)
-                                mark = zone->pages_min;
-                        else if (alloc_flags & ALLOC_WMARK_LOW)
-                                mark = zone->pages_low;
-                        else
-                                mark = zone->pages_high;
-                        if (!zone_watermark_ok(zone, order, mark,
-                                    classzone_idx, alloc_flags)) {
-                                if (!zone_reclaim_mode ||
-                                    !zone_reclaim(zone, gfp_mask, order))
+                        int ret;
+
+                        mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
+                        if (zone_watermark_ok(zone, order, mark,
+                                    classzone_idx, alloc_flags))
+                                goto try_this_zone;
+
+                        if (zone_reclaim_mode == 0)
+                                goto this_zone_full;
+
+                        ret = zone_reclaim(zone, gfp_mask, order);
+                        switch (ret) {
+                        case ZONE_RECLAIM_NOSCAN:
+                                /* did not scan */
+                                goto try_next_zone;
+                        case ZONE_RECLAIM_FULL:
+                                /* scanned but unreclaimable */
+                                goto this_zone_full;
+                        default:
+                                /* did we reclaim enough */
+                                if (!zone_watermark_ok(zone, order, mark,
+                                                classzone_idx, alloc_flags))
                                         goto this_zone_full;
                         }
                 }
 
-                page = buffered_rmqueue(preferred_zone, zone, order, gfp_mask);
+try_this_zone:
+                page = buffered_rmqueue(preferred_zone, zone, order,
+                                                gfp_mask, migratetype);
                 if (page)
                         break;
 this_zone_full:
                 if (NUMA_BUILD)
                         zlc_mark_zone_full(zonelist, z);
 try_next_zone:
-                if (NUMA_BUILD && !did_zlc_setup) {
-                        /* we do zlc_setup after the first zone is tried */
+                if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
+                        /*
+                         * we do zlc_setup after the first zone is tried but only
+                         * if there are multiple nodes make it worthwhile
+                         */
                         allowednodes = zlc_setup(zonelist, alloc_flags);
                         zlc_active = 1;
                         did_zlc_setup = 1;
@@ -1460,45 +1538,219 @@ try_next_zone:
         return page;
 }
 
+static inline int
+should_alloc_retry(gfp_t gfp_mask, unsigned int order,
+                                unsigned long pages_reclaimed)
+{
+        /* Do not loop if specifically requested */
+        if (gfp_mask & __GFP_NORETRY)
+                return 0;
+
+        /*
+         * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
+         * means __GFP_NOFAIL, but that may not be true in other
+         * implementations.
+         */
+        if (order <= PAGE_ALLOC_COSTLY_ORDER)
+                return 1;
+
+        /*
+         * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
+         * specified, then we retry until we no longer reclaim any pages
+         * (above), or we've reclaimed an order of pages at least as
+         * large as the allocation's order. In both cases, if the
+         * allocation still fails, we stop retrying.
+         */
+        if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
+                return 1;
+
+        /*
+         * Don't let big-order allocations loop unless the caller
+         * explicitly requests that.
+         */
+        if (gfp_mask & __GFP_NOFAIL)
+                return 1;
+
+        return 0;
+}
+
+static inline struct page *
+__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, struct zone *preferred_zone,
+        int migratetype)
+{
+        struct page *page;
+
+        /* Acquire the OOM killer lock for the zones in zonelist */
+        if (!try_set_zone_oom(zonelist, gfp_mask)) {
+                schedule_timeout_uninterruptible(1);
+                return NULL;
+        }
+
+        /*
+         * Go through the zonelist yet one more time, keep very high watermark
+         * here, this is only to catch a parallel oom killing, we must fail if
+         * we're still under heavy pressure.
+         */
+        page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
+                order, zonelist, high_zoneidx,
+                ALLOC_WMARK_HIGH|ALLOC_CPUSET,
+                preferred_zone, migratetype);
+        if (page)
+                goto out;
+
+        /* The OOM killer will not help higher order allocs */
+        if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_NOFAIL))
+                goto out;
+
+        /* Exhausted what can be done so it's blamo time */
+        out_of_memory(zonelist, gfp_mask, order);
+
+out:
+        clear_zonelist_oom(zonelist, gfp_mask);
+        return page;
+}
+
+/* The really slow allocator path where we enter direct reclaim */
+static inline struct page *
+__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
+        int migratetype, unsigned long *did_some_progress)
+{
+        struct page *page = NULL;
+        struct reclaim_state reclaim_state;
+        struct task_struct *p = current;
+
+        cond_resched();
+
+        /* We now go into synchronous reclaim */
+        cpuset_memory_pressure_bump();
+
+        /*
+         * The task's cpuset might have expanded its set of allowable nodes
+         */
+        p->flags |= PF_MEMALLOC;
+        lockdep_set_current_reclaim_state(gfp_mask);
+        reclaim_state.reclaimed_slab = 0;
+        p->reclaim_state = &reclaim_state;
+
+        *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
+
+        p->reclaim_state = NULL;
+        lockdep_clear_current_reclaim_state();
+        p->flags &= ~PF_MEMALLOC;
+
+        cond_resched();
+
+        if (order != 0)
+                drain_all_pages();
+
+        if (likely(*did_some_progress))
+                page = get_page_from_freelist(gfp_mask, nodemask, order,
+                                        zonelist, high_zoneidx,
+                                        alloc_flags, preferred_zone,
+                                        migratetype);
+        return page;
+}
+
 /*
- * This is the 'heart' of the zoned buddy allocator.
+ * This is called in the allocator slow-path if the allocation request is of
+ * sufficient urgency to ignore watermarks and take other desperate measures
  */
-struct page *
-__alloc_pages_internal(gfp_t gfp_mask, unsigned int order,
-                        struct zonelist *zonelist, nodemask_t *nodemask)
+static inline struct page *
+__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, struct zone *preferred_zone,
+        int migratetype)
+{
+        struct page *page;
+
+        do {
+                page = get_page_from_freelist(gfp_mask, nodemask, order,
+                        zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
+                        preferred_zone, migratetype);
+
+                if (!page && gfp_mask & __GFP_NOFAIL)
+                        congestion_wait(BLK_RW_ASYNC, HZ/50);
+        } while (!page && (gfp_mask & __GFP_NOFAIL));
+
+        return page;
+}
+
+static inline
+void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
+                                                enum zone_type high_zoneidx)
 {
-        const gfp_t wait = gfp_mask & __GFP_WAIT;
-        enum zone_type high_zoneidx = gfp_zone(gfp_mask);
         struct zoneref *z;
         struct zone *zone;
-        struct page *page;
-        struct reclaim_state reclaim_state;
-        struct task_struct *p = current;
-        int do_retry;
-        int alloc_flags;
-        unsigned long did_some_progress;
-        unsigned long pages_reclaimed = 0;
 
-        might_sleep_if(wait);
+        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
+                wakeup_kswapd(zone, order);
+}
 
-        if (should_fail_alloc_page(gfp_mask, order))
-                return NULL;
+static inline int
+gfp_to_alloc_flags(gfp_t gfp_mask)
+{
+        struct task_struct *p = current;
+        int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
+        const gfp_t wait = gfp_mask & __GFP_WAIT;
 
-restart:
-        z = zonelist->_zonerefs;  /* the list of zones suitable for gfp_mask */
+        /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
+        BUILD_BUG_ON(__GFP_HIGH != ALLOC_HIGH);
 
-        if (unlikely(!z->zone)) {
+        /*
+         * The caller may dip into page reserves a bit more if the caller
+         * cannot run direct reclaim, or if the caller has realtime scheduling
+         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
+         * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
+         */
+        alloc_flags |= (gfp_mask & __GFP_HIGH);
+
+        if (!wait) {
+                alloc_flags |= ALLOC_HARDER;
                 /*
-                 * Happens if we have an empty zonelist as a result of
-                 * GFP_THISNODE being used on a memoryless node
+                 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
+                 * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
                  */
-                return NULL;
+                alloc_flags &= ~ALLOC_CPUSET;
+        } else if (unlikely(rt_task(p)))
+                alloc_flags |= ALLOC_HARDER;
+
+        if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
+                if (!in_interrupt() &&
+                    ((p->flags & PF_MEMALLOC) ||
+                     unlikely(test_thread_flag(TIF_MEMDIE))))
+                        alloc_flags |= ALLOC_NO_WATERMARKS;
         }
 
-        page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
-                        zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET);
-        if (page)
-                goto got_pg;
+        return alloc_flags;
+}
+
+static inline struct page *
+__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, struct zone *preferred_zone,
+        int migratetype)
+{
+        const gfp_t wait = gfp_mask & __GFP_WAIT;
+        struct page *page = NULL;
+        int alloc_flags;
+        unsigned long pages_reclaimed = 0;
+        unsigned long did_some_progress;
+        struct task_struct *p = current;
+
+        /*
+         * In the slowpath, we sanity check order to avoid ever trying to
+         * reclaim >= MAX_ORDER areas which will never succeed. Callers may
+         * be using allocators in order of preference for an area that is
+         * too large.
+         */
+        if (order >= MAX_ORDER) {
+                WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
+                return NULL;
+        }
 
         /*
          * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
@@ -1511,151 +1763,88 @@ restart:
         if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
                 goto nopage;
 
-        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
-                wakeup_kswapd(zone, order);
+        wake_all_kswapd(order, zonelist, high_zoneidx);
 
         /*
          * OK, we're below the kswapd watermark and have kicked background
          * reclaim. Now things get more complex, so set up alloc_flags according
          * to how we want to proceed.
-         *
-         * The caller may dip into page reserves a bit more if the caller
-         * cannot run direct reclaim, or if the caller has realtime scheduling
-         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
-         * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
          */
-        alloc_flags = ALLOC_WMARK_MIN;
-        if ((unlikely(rt_task(p)) && !in_interrupt()) || !wait)
-                alloc_flags |= ALLOC_HARDER;
-        if (gfp_mask & __GFP_HIGH)
-                alloc_flags |= ALLOC_HIGH;
-        if (wait)
-                alloc_flags |= ALLOC_CPUSET;
+        alloc_flags = gfp_to_alloc_flags(gfp_mask);
 
-        /*
-         * Go through the zonelist again. Let __GFP_HIGH and allocations
-         * coming from realtime tasks go deeper into reserves.
-         *
-         * This is the last chance, in general, before the goto nopage.
-         * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
-         * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
-         */
+restart:
+        /* This is the last chance, in general, before the goto nopage. */
         page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
-                                                high_zoneidx, alloc_flags);
+                        high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
+                        preferred_zone, migratetype);
         if (page)
                 goto got_pg;
 
-        /* This allocation should allow future memory freeing. */
-
 rebalance:
-        if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE)))
-                        && !in_interrupt()) {
-                if (!(gfp_mask & __GFP_NOMEMALLOC)) {
-nofail_alloc:
-                        /* go through the zonelist yet again, ignoring mins */
-                        page = get_page_from_freelist(gfp_mask, nodemask, order,
-                                zonelist, high_zoneidx, ALLOC_NO_WATERMARKS);
-                        if (page)
-                                goto got_pg;
-                        if (gfp_mask & __GFP_NOFAIL) {
-                                congestion_wait(WRITE, HZ/50);
-                                goto nofail_alloc;
-                        }
-                }
-                goto nopage;
+        /* Allocate without watermarks if the context allows */
+        if (alloc_flags & ALLOC_NO_WATERMARKS) {
+                page = __alloc_pages_high_priority(gfp_mask, order,
+                                zonelist, high_zoneidx, nodemask,
+                                preferred_zone, migratetype);
+                if (page)
+                        goto got_pg;
         }
 
         /* Atomic allocations - we can't balance anything */
         if (!wait)
                 goto nopage;
 
-        cond_resched();
-
-        /* We now go into synchronous reclaim */
-        cpuset_memory_pressure_bump();
-        /*
-         * The task's cpuset might have expanded its set of allowable nodes
-         */
-        cpuset_update_task_memory_state();
-        p->flags |= PF_MEMALLOC;
-        reclaim_state.reclaimed_slab = 0;
-        p->reclaim_state = &reclaim_state;
+        /* Avoid recursion of direct reclaim */
+        if (p->flags & PF_MEMALLOC)
+                goto nopage;
 
-        did_some_progress = try_to_free_pages(zonelist, order, gfp_mask);
+        /* Avoid allocations with no watermarks from looping endlessly */
+        if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
+                goto nopage;
 
-        p->reclaim_state = NULL;
-        p->flags &= ~PF_MEMALLOC;
+        /* Try direct reclaim and then allocating */
+        page = __alloc_pages_direct_reclaim(gfp_mask, order,
+                                        zonelist, high_zoneidx,
+                                        nodemask,
+                                        alloc_flags, preferred_zone,
+                                        migratetype, &did_some_progress);
+        if (page)
+                goto got_pg;
 
-        cond_resched();
+        /*
+         * If we failed to make any progress reclaiming, then we are
+         * running out of options and have to consider going OOM
+         */
+        if (!did_some_progress) {
+                if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
+                        if (oom_killer_disabled)
+                                goto nopage;
+                        page = __alloc_pages_may_oom(gfp_mask, order,
+                                        zonelist, high_zoneidx,
+                                        nodemask, preferred_zone,
+                                        migratetype);
+                        if (page)
+                                goto got_pg;
 
-        if (order != 0)
-                drain_all_pages();
+                        /*
+                         * The OOM killer does not trigger for high-order
+                         * ~__GFP_NOFAIL allocations so if no progress is being
+                         * made, there are no other options and retrying is
+                         * unlikely to help.
+                         */
+                        if (order > PAGE_ALLOC_COSTLY_ORDER &&
+                                                !(gfp_mask & __GFP_NOFAIL))
+                                goto nopage;
 
-        if (likely(did_some_progress)) {
-                page = get_page_from_freelist(gfp_mask, nodemask, order,
-                                        zonelist, high_zoneidx, alloc_flags);
-                if (page)
-                        goto got_pg;
-        } else if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
-                if (!try_set_zone_oom(zonelist, gfp_mask)) {
-                        schedule_timeout_uninterruptible(1);
                         goto restart;
                 }
-
-                /*
-                 * Go through the zonelist yet one more time, keep
-                 * very high watermark here, this is only to catch
-                 * a parallel oom killing, we must fail if we're still
-                 * under heavy pressure.
-                 */
-                page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
-                        order, zonelist, high_zoneidx,
-                        ALLOC_WMARK_HIGH|ALLOC_CPUSET);
-                if (page) {
-                        clear_zonelist_oom(zonelist, gfp_mask);
-                        goto got_pg;
-                }
-
-                /* The OOM killer will not help higher order allocs so fail */
-                if (order > PAGE_ALLOC_COSTLY_ORDER) {
-                        clear_zonelist_oom(zonelist, gfp_mask);
-                        goto nopage;
-                }
-
-                out_of_memory(zonelist, gfp_mask, order);
-                clear_zonelist_oom(zonelist, gfp_mask);
-                goto restart;
         }
 
-        /*
-         * Don't let big-order allocations loop unless the caller explicitly
-         * requests that.  Wait for some write requests to complete then retry.
-         *
-         * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
-         * means __GFP_NOFAIL, but that may not be true in other
-         * implementations.
-         *
-         * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
-         * specified, then we retry until we no longer reclaim any pages
-         * (above), or we've reclaimed an order of pages at least as
-         * large as the allocation's order. In both cases, if the
-         * allocation still fails, we stop retrying.
-         */
+        /* Check if we should retry the allocation */
         pages_reclaimed += did_some_progress;
-        do_retry = 0;
-        if (!(gfp_mask & __GFP_NORETRY)) {
-                if (order <= PAGE_ALLOC_COSTLY_ORDER) {
-                        do_retry = 1;
-                } else {
-                        if (gfp_mask & __GFP_REPEAT &&
-                                pages_reclaimed < (1 << order))
-                                        do_retry = 1;
-                }
-                if (gfp_mask & __GFP_NOFAIL)
-                        do_retry = 1;
-        }
-        if (do_retry) {
-                congestion_wait(WRITE, HZ/50);
+        if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) {
+                /* Wait for some write requests to complete then retry */
+                congestion_wait(BLK_RW_ASYNC, HZ/50);
                 goto rebalance;
         }
 
@@ -1667,10 +1856,60 @@ nopage:
                 dump_stack();
                 show_mem();
         }
+        return page;
 got_pg:
+        if (kmemcheck_enabled)
+                kmemcheck_pagealloc_alloc(page, order, gfp_mask);
         return page;
+
 }
-EXPORT_SYMBOL(__alloc_pages_internal);
+
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *
+__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
+                        struct zonelist *zonelist, nodemask_t *nodemask)
+{
+        enum zone_type high_zoneidx = gfp_zone(gfp_mask);
+        struct zone *preferred_zone;
+        struct page *page;
+        int migratetype = allocflags_to_migratetype(gfp_mask);
+
+        gfp_mask &= gfp_allowed_mask;
+
+        lockdep_trace_alloc(gfp_mask);
+
+        might_sleep_if(gfp_mask & __GFP_WAIT);
+
+        if (should_fail_alloc_page(gfp_mask, order))
+                return NULL;
+
+        /*
+         * Check the zones suitable for the gfp_mask contain at least one
+         * valid zone. It's possible to have an empty zonelist as a result
+         * of GFP_THISNODE and a memoryless node
+         */
+        if (unlikely(!zonelist->_zonerefs->zone))
+                return NULL;
+
+        /* The preferred zone is used for statistics later */
+        first_zones_zonelist(zonelist, high_zoneidx, nodemask, &preferred_zone);
+        if (!preferred_zone)
+                return NULL;
+
+        /* First allocation attempt */
+        page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
+                        zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
+                        preferred_zone, migratetype);
+        if (unlikely(!page))
+                page = __alloc_pages_slowpath(gfp_mask, order,
+                                zonelist, high_zoneidx, nodemask,
+                                preferred_zone, migratetype);
+
+        return page;
+}
+EXPORT_SYMBOL(__alloc_pages_nodemask);
 
 /*
  * Common helper functions.
@@ -1757,7 +1996,7 @@ void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
                 unsigned long alloc_end = addr + (PAGE_SIZE << order);
                 unsigned long used = addr + PAGE_ALIGN(size);
 
-                split_page(virt_to_page(addr), order);
+                split_page(virt_to_page((void *)addr), order);
                 while (used < alloc_end) {
                         free_page(used);
                         used += PAGE_SIZE;
@@ -1799,7 +2038,7 @@ static unsigned int nr_free_zone_pages(int offset)
 
         for_each_zone_zonelist(zone, z, zonelist, offset) {
                 unsigned long size = zone->present_pages;
-                unsigned long high = zone->pages_high;
+                unsigned long high = high_wmark_pages(zone);
                 if (size > high)
                         sum += size - high;
         }
@@ -1874,10 +2113,7 @@ void show_free_areas(void)
         int cpu;
         struct zone *zone;
 
-        for_each_zone(zone) {
-                if (!populated_zone(zone))
-                        continue;
-
+        for_each_populated_zone(zone) {
                 show_node(zone);
                 printk("%s per-cpu:\n", zone->name);
 
@@ -1894,19 +2130,14 @@ void show_free_areas(void)
 
         printk("Active_anon:%lu active_file:%lu inactive_anon:%lu\n"
                 " inactive_file:%lu"
-//TODO:  check/adjust line lengths
-#ifdef CONFIG_UNEVICTABLE_LRU
                 " unevictable:%lu"
-#endif
                 " dirty:%lu writeback:%lu unstable:%lu\n"
                 " free:%lu slab:%lu mapped:%lu pagetables:%lu bounce:%lu\n",
                 global_page_state(NR_ACTIVE_ANON),
                 global_page_state(NR_ACTIVE_FILE),
                 global_page_state(NR_INACTIVE_ANON),
                 global_page_state(NR_INACTIVE_FILE),
-#ifdef CONFIG_UNEVICTABLE_LRU
                 global_page_state(NR_UNEVICTABLE),
-#endif
                 global_page_state(NR_FILE_DIRTY),
                 global_page_state(NR_WRITEBACK),
                 global_page_state(NR_UNSTABLE_NFS),
@@ -1917,12 +2148,9 @@ void show_free_areas(void)
                 global_page_state(NR_PAGETABLE),
                 global_page_state(NR_BOUNCE));
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 int i;
 
-                if (!populated_zone(zone))
-                        continue;
-
                 show_node(zone);
                 printk("%s"
                         " free:%lukB"
@@ -1933,25 +2161,21 @@ void show_free_areas(void)
                         " inactive_anon:%lukB"
                         " active_file:%lukB"
                         " inactive_file:%lukB"
-#ifdef CONFIG_UNEVICTABLE_LRU
                         " unevictable:%lukB"
-#endif
                         " present:%lukB"
                         " pages_scanned:%lu"
                         " all_unreclaimable? %s"
                         "\n",
                         zone->name,
                         K(zone_page_state(zone, NR_FREE_PAGES)),
-                        K(zone->pages_min),
-                        K(zone->pages_low),
-                        K(zone->pages_high),
+                        K(min_wmark_pages(zone)),
+                        K(low_wmark_pages(zone)),
+                        K(high_wmark_pages(zone)),
                         K(zone_page_state(zone, NR_ACTIVE_ANON)),
                         K(zone_page_state(zone, NR_INACTIVE_ANON)),
                         K(zone_page_state(zone, NR_ACTIVE_FILE)),
                         K(zone_page_state(zone, NR_INACTIVE_FILE)),
-#ifdef CONFIG_UNEVICTABLE_LRU
                         K(zone_page_state(zone, NR_UNEVICTABLE)),
-#endif
                         K(zone->present_pages),
                         zone->pages_scanned,
                         (zone_is_all_unreclaimable(zone) ? "yes" : "no")
@@ -1962,12 +2186,9 @@ void show_free_areas(void)
                 printk("\n");
         }
 
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 unsigned long nr[MAX_ORDER], flags, order, total = 0;
 
-                if (!populated_zone(zone))
-                        continue;
-
                 show_node(zone);
                 printk("%s: ", zone->name);
 
@@ -2112,7 +2333,7 @@ int numa_zonelist_order_handler(ctl_table *table, int write,
 }
 
 
-#define MAX_NODE_LOAD (num_online_nodes())
+#define MAX_NODE_LOAD (nr_online_nodes)
 static int node_load[MAX_NUMNODES];
 
 /**
@@ -2134,7 +2355,7 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
         int n, val;
         int min_val = INT_MAX;
         int best_node = -1;
-        node_to_cpumask_ptr(tmp, 0);
+        const struct cpumask *tmp = cpumask_of_node(0);
 
         /* Use the local node if we haven't already */
         if (!node_isset(node, *used_node_mask)) {
@@ -2155,8 +2376,8 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
                 val += (n < node);
 
                 /* Give preference to headless and unused nodes */
-                node_to_cpumask_ptr_next(tmp, n);
-                if (!cpus_empty(*tmp))
+                tmp = cpumask_of_node(n);
+                if (!cpumask_empty(tmp))
                         val += PENALTY_FOR_NODE_WITH_CPUS;
 
                 /* Slight preference for less loaded node */
@@ -2321,11 +2542,10 @@ static void build_zonelists(pg_data_t *pgdat)
 
         /* NUMA-aware ordering of nodes */
         local_node = pgdat->node_id;
-        load = num_online_nodes();
+        load = nr_online_nodes;
         prev_node = local_node;
         nodes_clear(used_mask);
 
-        memset(node_load, 0, sizeof(node_load));
         memset(node_order, 0, sizeof(node_order));
         j = 0;
 
@@ -2434,6 +2654,9 @@ static int __build_all_zonelists(void *dummy)
 {
         int nid;
 
+#ifdef CONFIG_NUMA
+        memset(node_load, 0, sizeof(node_load));
+#endif
         for_each_online_node(nid) {
                 pg_data_t *pgdat = NODE_DATA(nid);
 
@@ -2472,7 +2695,7 @@ void build_all_zonelists(void)
 
         printk("Built %i zonelists in %s order, mobility grouping %s.  "
                 "Total pages: %ld\n",
-                        num_online_nodes(),
+                        nr_online_nodes,
                         zonelist_order_name[current_zonelist_order],
                         page_group_by_mobility_disabled ? "off" : "on",
                         vm_total_pages);
@@ -2551,8 +2774,8 @@ static inline unsigned long wait_table_bits(unsigned long size)
 
 /*
  * Mark a number of pageblocks as MIGRATE_RESERVE. The number
- * of blocks reserved is based on zone->pages_min. The memory within the
- * reserve will tend to store contiguous free pages. Setting min_free_kbytes
+ * of blocks reserved is based on min_wmark_pages(zone). The memory within
+ * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  * higher will lead to a bigger reserve which will get freed as contiguous
  * blocks as reclaim kicks in
  */
@@ -2565,7 +2788,7 @@ static void setup_zone_migrate_reserve(struct zone *zone)
         /* Get the start pfn, end pfn and the number of blocks to reserve */
         start_pfn = zone->zone_start_pfn;
         end_pfn = start_pfn + zone->spanned_pages;
-        reserve = roundup(zone->pages_min, pageblock_nr_pages) >>
+        reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
                                                         pageblock_order;
 
         for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
@@ -2687,6 +2910,7 @@ static void __meminit zone_init_free_lists(struct zone *zone)
 
 static int zone_batchsize(struct zone *zone)
 {
+#ifdef CONFIG_MMU
         int batch;
 
         /*
@@ -2712,9 +2936,26 @@ static int zone_batchsize(struct zone *zone)
          * of pages of one half of the possible page colors
          * and the other with pages of the other colors.
          */
-        batch = (1 << (fls(batch + batch/2)-1)) - 1;
+        batch = rounddown_pow_of_two(batch + batch/2) - 1;
 
         return batch;
+
+#else
+        /* The deferral and batching of frees should be suppressed under NOMMU
+         * conditions.
+         *
+         * The problem is that NOMMU needs to be able to allocate large chunks
+         * of contiguous memory as there's no hardware page translation to
+         * assemble apparent contiguous memory from discontiguous pages.
+         *
+         * Queueing large contiguous runs of pages for batching, however,
+         * causes the pages to actually be freed in smaller chunks.  As there
+         * can be a significant delay between the individual batches being
+         * recycled, this leads to the once large chunks of space being
+         * fragmented and becoming unavailable for high-order allocations.
+         */
+        return 0;
+#endif
 }
 
 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
@@ -2779,11 +3020,7 @@ static int __cpuinit process_zones(int cpu)
 
         node_set_state(node, N_CPU);    /* this node has a cpu */
 
-        for_each_zone(zone) {
-
-                if (!populated_zone(zone))
-                        continue;
-
+        for_each_populated_zone(zone) {
                 zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset),
                                          GFP_KERNEL, node);
                 if (!zone_pcp(zone, cpu))
@@ -2804,7 +3041,7 @@ bad:
                 if (dzone == zone)
                         break;
                 kfree(zone_pcp(dzone, cpu));
-                zone_pcp(dzone, cpu) = NULL;
+                zone_pcp(dzone, cpu) = &boot_pageset[cpu];
         }
         return -ENOMEM;
 }
@@ -2819,7 +3056,7 @@ static inline void free_zone_pagesets(int cpu)
                 /* Free per_cpu_pageset if it is slab allocated */
                 if (pset != &boot_pageset[cpu])
                         kfree(pset);
-                zone_pcp(zone, cpu) = NULL;
+                zone_pcp(zone, cpu) = &boot_pageset[cpu];
         }
 }
 
@@ -3095,64 +3332,6 @@ void __init sparse_memory_present_with_active_regions(int nid)
 }
 
 /**
- * push_node_boundaries - Push node boundaries to at least the requested boundary
- * @nid: The nid of the node to push the boundary for
- * @start_pfn: The start pfn of the node
- * @end_pfn: The end pfn of the node
- *
- * In reserve-based hot-add, mem_map is allocated that is unused until hotadd
- * time. Specifically, on x86_64, SRAT will report ranges that can potentially
- * be hotplugged even though no physical memory exists. This function allows
- * an arch to push out the node boundaries so mem_map is allocated that can
- * be used later.
- */
-#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
-void __init push_node_boundaries(unsigned int nid,
-                unsigned long start_pfn, unsigned long end_pfn)
-{
-        mminit_dprintk(MMINIT_TRACE, "zoneboundary",
-                        "Entering push_node_boundaries(%u, %lu, %lu)\n",
-                        nid, start_pfn, end_pfn);
-
-        /* Initialise the boundary for this node if necessary */
-        if (node_boundary_end_pfn[nid] == 0)
-                node_boundary_start_pfn[nid] = -1UL;
-
-        /* Update the boundaries */
-        if (node_boundary_start_pfn[nid] > start_pfn)
-                node_boundary_start_pfn[nid] = start_pfn;
-        if (node_boundary_end_pfn[nid] < end_pfn)
-                node_boundary_end_pfn[nid] = end_pfn;
-}
-
-/* If necessary, push the node boundary out for reserve hotadd */
-static void __meminit account_node_boundary(unsigned int nid,
-                unsigned long *start_pfn, unsigned long *end_pfn)
-{
-        mminit_dprintk(MMINIT_TRACE, "zoneboundary",
-                        "Entering account_node_boundary(%u, %lu, %lu)\n",
-                        nid, *start_pfn, *end_pfn);
-
-        /* Return if boundary information has not been provided */
-        if (node_boundary_end_pfn[nid] == 0)
-                return;
-
-        /* Check the boundaries and update if necessary */
-        if (node_boundary_start_pfn[nid] < *start_pfn)
-                *start_pfn = node_boundary_start_pfn[nid];
-        if (node_boundary_end_pfn[nid] > *end_pfn)
-                *end_pfn = node_boundary_end_pfn[nid];
-}
-#else
-void __init push_node_boundaries(unsigned int nid,
-                unsigned long start_pfn, unsigned long end_pfn) {}
-
-static void __meminit account_node_boundary(unsigned int nid,
-                unsigned long *start_pfn, unsigned long *end_pfn) {}
-#endif
-
-
-/**
  * get_pfn_range_for_nid - Return the start and end page frames for a node
  * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
@@ -3177,9 +3356,6 @@ void __meminit get_pfn_range_for_nid(unsigned int nid,
 
         if (*start_pfn == -1UL)
                 *start_pfn = 0;
-
-        /* Push the node boundaries out if requested */
-        account_node_boundary(nid, start_pfn, end_pfn);
 }
 
 /*
@@ -3544,7 +3720,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                 zone_pcp_init(zone);
                 for_each_lru(l) {
                         INIT_LIST_HEAD(&zone->lru[l].list);
-                        zone->lru[l].nr_scan = 0;
+                        zone->lru[l].nr_saved_scan = 0;
                 }
                 zone->reclaim_stat.recent_rotated[0] = 0;
                 zone->reclaim_stat.recent_rotated[1] = 0;
@@ -3785,10 +3961,6 @@ void __init remove_all_active_ranges(void)
 {
         memset(early_node_map, 0, sizeof(early_node_map));
         nr_nodemap_entries = 0;
-#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
-        memset(node_boundary_start_pfn, 0, sizeof(node_boundary_start_pfn));
-        memset(node_boundary_end_pfn, 0, sizeof(node_boundary_end_pfn));
-#endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */
 }
 
 /* Compare two active node_active_regions */
@@ -3875,6 +4047,8 @@ static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
         int i, nid;
         unsigned long usable_startpfn;
         unsigned long kernelcore_node, kernelcore_remaining;
+        /* save the state before borrow the nodemask */
+        nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
         unsigned long totalpages = early_calculate_totalpages();
         int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
 
@@ -3902,7 +4076,7 @@ static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
 
         /* If kernelcore was not specified, there is no ZONE_MOVABLE */
         if (!required_kernelcore)
-                return;
+                goto out;
 
         /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
         find_usable_zone_for_movable();
@@ -4001,6 +4175,10 @@ restart:
         for (nid = 0; nid < MAX_NUMNODES; nid++)
                 zone_movable_pfn[nid] =
                         roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
+
+out:
+        /* restore the node_state */
+        node_states[N_HIGH_MEMORY] = saved_node_state;
 }
 
 /* Any regular memory on that node ? */
@@ -4219,8 +4397,8 @@ static void calculate_totalreserve_pages(void)
                                         max = zone->lowmem_reserve[j];
                         }
 
-                        /* we treat pages_high as reserved pages. */
-                        max += zone->pages_high;
+                        /* we treat the high watermark as reserved pages. */
+                        max += high_wmark_pages(zone);
 
                         if (max > zone->present_pages)
                                 max = zone->present_pages;
@@ -4270,12 +4448,13 @@ static void setup_per_zone_lowmem_reserve(void)
 }
 
 /**
- * setup_per_zone_pages_min - called when min_free_kbytes changes.
+ * setup_per_zone_wmarks - called when min_free_kbytes changes
+ * or when memory is hot-{added|removed}
  *
- * Ensures that the pages_{min,low,high} values for each zone are set correctly
- * with respect to min_free_kbytes.
+ * Ensures that the watermark[min,low,high] values for each zone are set
+ * correctly with respect to min_free_kbytes.
  */
-void setup_per_zone_pages_min(void)
+void setup_per_zone_wmarks(void)
 {
         unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
         unsigned long lowmem_pages = 0;
@@ -4300,7 +4479,7 @@ void setup_per_zone_pages_min(void)
                          * need highmem pages, so cap pages_min to a small
                          * value here.
                          *
-                         * The (pages_high-pages_low) and (pages_low-pages_min)
+                         * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
                          * deltas controls asynch page reclaim, and so should
                          * not be capped for highmem.
                          */
@@ -4311,17 +4490,17 @@ void setup_per_zone_pages_min(void)
                                 min_pages = SWAP_CLUSTER_MAX;
                         if (min_pages > 128)
                                 min_pages = 128;
-                        zone->pages_min = min_pages;
+                        zone->watermark[WMARK_MIN] = min_pages;
                 } else {
                         /*
                          * If it's a lowmem zone, reserve a number of pages
                          * proportionate to the zone's size.
                          */
-                        zone->pages_min = tmp;
+                        zone->watermark[WMARK_MIN] = tmp;
                 }
 
-                zone->pages_low   = zone->pages_min + (tmp >> 2);
-                zone->pages_high  = zone->pages_min + (tmp >> 1);
+                zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + (tmp >> 2);
+                zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
                 setup_zone_migrate_reserve(zone);
                 spin_unlock_irqrestore(&zone->lock, flags);
         }
@@ -4331,8 +4510,6 @@ void setup_per_zone_pages_min(void)
 }
 
 /**
- * setup_per_zone_inactive_ratio - called when min_free_kbytes changes.
- *
  * The inactive anon list should be small enough that the VM never has to
  * do too much work, but large enough that each inactive page has a chance
  * to be referenced again before it is swapped out.
@@ -4353,21 +4530,26 @@ void setup_per_zone_pages_min(void)
  *    1TB     101        10GB
  *   10TB     320        32GB
  */
-static void setup_per_zone_inactive_ratio(void)
+void calculate_zone_inactive_ratio(struct zone *zone)
 {
-        struct zone *zone;
+        unsigned int gb, ratio;
 
-        for_each_zone(zone) {
-                unsigned int gb, ratio;
-
-                /* Zone size in gigabytes */
-                gb = zone->present_pages >> (30 - PAGE_SHIFT);
+        /* Zone size in gigabytes */
+        gb = zone->present_pages >> (30 - PAGE_SHIFT);
+        if (gb)
                 ratio = int_sqrt(10 * gb);
-                if (!ratio)
-                        ratio = 1;
+        else
+                ratio = 1;
 
-                zone->inactive_ratio = ratio;
-        }
+        zone->inactive_ratio = ratio;
+}
+
+static void __init setup_per_zone_inactive_ratio(void)
+{
+        struct zone *zone;
+
+        for_each_zone(zone)
+                calculate_zone_inactive_ratio(zone);
 }
 
 /*
@@ -4394,7 +4576,7 @@ static void setup_per_zone_inactive_ratio(void)
  * 8192MB:      11584k
  * 16384MB:     16384k
  */
-static int __init init_per_zone_pages_min(void)
+static int __init init_per_zone_wmark_min(void)
 {
         unsigned long lowmem_kbytes;
 
@@ -4405,12 +4587,12 @@ static int __init init_per_zone_pages_min(void)
                 min_free_kbytes = 128;
         if (min_free_kbytes > 65536)
                 min_free_kbytes = 65536;
-        setup_per_zone_pages_min();
+        setup_per_zone_wmarks();
         setup_per_zone_lowmem_reserve();
         setup_per_zone_inactive_ratio();
         return 0;
 }
-module_init(init_per_zone_pages_min)
+module_init(init_per_zone_wmark_min)
 
 /*
  * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 
@@ -4422,7 +4604,7 @@ int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
 {
         proc_dointvec(table, write, file, buffer, length, ppos);
         if (write)
-                setup_per_zone_pages_min();
+                setup_per_zone_wmarks();
         return 0;
 }
 
@@ -4466,7 +4648,7 @@ int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  *      whenever sysctl_lowmem_reserve_ratio changes.
  *
  * The reserve ratio obviously has absolutely no relation with the
- * pages_min watermarks. The lowmem reserve ratio can only make sense
+ * minimum watermarks. The lowmem reserve ratio can only make sense
  * if in function of the boot time zone sizes.
  */
 int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
@@ -4493,7 +4675,7 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
         ret = proc_dointvec_minmax(table, write, file, buffer, length, ppos);
         if (!write || (ret == -EINVAL))
                 return ret;
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 for_each_online_cpu(cpu) {
                         unsigned long  high;
                         high = zone->present_pages / percpu_pagelist_fraction;
@@ -4573,22 +4755,14 @@ void *__init alloc_large_system_hash(const char *tablename,
                 else if (hashdist)
                         table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
                 else {
-                        unsigned long order = get_order(size);
-                        table = (void*) __get_free_pages(GFP_ATOMIC, order);
                         /*
                          * If bucketsize is not a power-of-two, we may free
-                         * some pages at the end of hash table.
+                         * some pages at the end of hash table which
+                         * alloc_pages_exact() automatically does
                          */
-                        if (table) {
-                                unsigned long alloc_end = (unsigned long)table +
-                                                (PAGE_SIZE << order);
-                                unsigned long used = (unsigned long)table +
-                                                PAGE_ALIGN(size);
-                                split_page(virt_to_page(table), order);
-                                while (used < alloc_end) {
-                                        free_page(used);
-                                        used += PAGE_SIZE;
-                                }
+                        if (get_order(size) < MAX_ORDER) {
+                                table = alloc_pages_exact(size, GFP_ATOMIC);
+                                kmemleak_alloc(table, size, 1, GFP_ATOMIC);
                         }
                 }
         } while (!table && size > PAGE_SIZE && --log2qty);