1 files changed, 660 insertions, 71 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 1a8c59571cb7..d315e1127dc9 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -41,24 +41,37 @@
 #include <linux/pfn.h>
 #include <linux/backing-dev.h>
 #include <linux/fault-inject.h>
+#include <linux/page-isolation.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>
 #include "internal.h"
 
 /*
- * MCD - HACK: Find somewhere to initialize this EARLY, or make this
- * initializer cleaner
+ * Array of node states.
  */
-nodemask_t node_online_map __read_mostly = { { [0] = 1UL } };
-EXPORT_SYMBOL(node_online_map);
-nodemask_t node_possible_map __read_mostly = NODE_MASK_ALL;
-EXPORT_SYMBOL(node_possible_map);
+nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
+        [N_POSSIBLE] = NODE_MASK_ALL,
+        [N_ONLINE] = { { [0] = 1UL } },
+#ifndef CONFIG_NUMA
+        [N_NORMAL_MEMORY] = { { [0] = 1UL } },
+#ifdef CONFIG_HIGHMEM
+        [N_HIGH_MEMORY] = { { [0] = 1UL } },
+#endif
+        [N_CPU] = { { [0] = 1UL } },
+#endif  /* NUMA */
+};
+EXPORT_SYMBOL(node_states);
+
 unsigned long totalram_pages __read_mostly;
 unsigned long totalreserve_pages __read_mostly;
 long nr_swap_pages;
 int percpu_pagelist_fraction;
 
+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
+int pageblock_order __read_mostly;
+#endif
+
 static void __free_pages_ok(struct page *page, unsigned int order);
 
 /*
@@ -137,7 +150,7 @@ static unsigned long __meminitdata dma_reserve;
   static unsigned long __meminitdata node_boundary_end_pfn[MAX_NUMNODES];
 #endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */
   unsigned long __initdata required_kernelcore;
-  unsigned long __initdata required_movablecore;
+  static unsigned long __initdata required_movablecore;
   unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
 
   /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
@@ -150,6 +163,14 @@ int nr_node_ids __read_mostly = MAX_NUMNODES;
 EXPORT_SYMBOL(nr_node_ids);
 #endif
 
+int page_group_by_mobility_disabled __read_mostly;
+
+static void set_pageblock_migratetype(struct page *page, int migratetype)
+{
+        set_pageblock_flags_group(page, (unsigned long)migratetype,
+                                        PB_migrate, PB_migrate_end);
+}
+
 #ifdef CONFIG_DEBUG_VM
 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 {
@@ -293,16 +314,6 @@ static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
                 clear_highpage(page + i);
 }
 
-/*
- * function for dealing with page's order in buddy system.
- * zone->lock is already acquired when we use these.
- * So, we don't need atomic page->flags operations here.
- */
-static inline unsigned long page_order(struct page *page)
-{
-        return page_private(page);
-}
-
 static inline void set_page_order(struct page *page, int order)
 {
         set_page_private(page, order);
@@ -404,6 +415,7 @@ static inline void __free_one_page(struct page *page,
 {
         unsigned long page_idx;
         int order_size = 1 << order;
+        int migratetype = get_pageblock_migratetype(page);
 
         if (unlikely(PageCompound(page)))
                 destroy_compound_page(page, order);
@@ -416,7 +428,6 @@ static inline void __free_one_page(struct page *page,
         __mod_zone_page_state(zone, NR_FREE_PAGES, order_size);
         while (order < MAX_ORDER-1) {
                 unsigned long combined_idx;
-                struct free_area *area;
                 struct page *buddy;
 
                 buddy = __page_find_buddy(page, page_idx, order);
@@ -424,8 +435,7 @@ static inline void __free_one_page(struct page *page,
                         break;          /* Move the buddy up one level. */
 
                 list_del(&buddy->lru);
-                area = zone->free_area + order;
-                area->nr_free--;
+                zone->free_area[order].nr_free--;
                 rmv_page_order(buddy);
                 combined_idx = __find_combined_index(page_idx, order);
                 page = page + (combined_idx - page_idx);
@@ -433,7 +443,8 @@ static inline void __free_one_page(struct page *page,
                 order++;
         }
         set_page_order(page, order);
-        list_add(&page->lru, &zone->free_area[order].free_list);
+        list_add(&page->lru,
+                &zone->free_area[order].free_list[migratetype]);
         zone->free_area[order].nr_free++;
 }
 
@@ -567,7 +578,8 @@ void fastcall __init __free_pages_bootmem(struct page *page, unsigned int order)
  * -- wli
  */
 static inline void expand(struct zone *zone, struct page *page,
-        int low, int high, struct free_area *area)
+        int low, int high, struct free_area *area,
+        int migratetype)
 {
         unsigned long size = 1 << high;
 
@@ -576,7 +588,7 @@ static inline void expand(struct zone *zone, struct page *page,
                 high--;
                 size >>= 1;
                 VM_BUG_ON(bad_range(zone, &page[size]));
-                list_add(&page[size].lru, &area->free_list);
+                list_add(&page[size].lru, &area->free_list[migratetype]);
                 area->nr_free++;
                 set_page_order(&page[size], high);
         }
@@ -628,49 +640,235 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
         return 0;
 }
 
-/* 
- * Do the hard work of removing an element from the buddy allocator.
- * Call me with the zone->lock already held.
+/*
+ * Go through the free lists for the given migratetype and remove
+ * the smallest available page from the freelists
  */
-static struct page *__rmqueue(struct zone *zone, unsigned int order)
+static struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
+                                                int migratetype)
 {
-        struct free_area * area;
         unsigned int current_order;
+        struct free_area * area;
         struct page *page;
 
+        /* Find a page of the appropriate size in the preferred list */
         for (current_order = order; current_order < MAX_ORDER; ++current_order) {
-                area = zone->free_area + current_order;
-                if (list_empty(&area->free_list))
+                area = &(zone->free_area[current_order]);
+                if (list_empty(&area->free_list[migratetype]))
                         continue;
 
-                page = list_entry(area->free_list.next, struct page, lru);
+                page = list_entry(area->free_list[migratetype].next,
+                                                        struct page, lru);
                 list_del(&page->lru);
                 rmv_page_order(page);
                 area->nr_free--;
                 __mod_zone_page_state(zone, NR_FREE_PAGES, - (1UL << order));
-                expand(zone, page, order, current_order, area);
+                expand(zone, page, order, current_order, area, migratetype);
                 return page;
         }
 
         return NULL;
 }
 
+
+/*
+ * This array describes the order lists are fallen back to when
+ * the free lists for the desirable migrate type are depleted
+ */
+static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
+        [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_RESERVE },
+        [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_RESERVE },
+        [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
+        [MIGRATE_RESERVE]     = { MIGRATE_RESERVE,     MIGRATE_RESERVE,   MIGRATE_RESERVE }, /* Never used */
+};
+
+/*
+ * Move the free pages in a range to the free lists of the requested type.
+ * Note that start_page and end_pages are not aligned on a pageblock
+ * boundary. If alignment is required, use move_freepages_block()
+ */
+int move_freepages(struct zone *zone,
+                        struct page *start_page, struct page *end_page,
+                        int migratetype)
+{
+        struct page *page;
+        unsigned long order;
+        int pages_moved = 0;
+
+#ifndef CONFIG_HOLES_IN_ZONE
+        /*
+         * page_zone is not safe to call in this context when
+         * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
+         * anyway as we check zone boundaries in move_freepages_block().
+         * Remove at a later date when no bug reports exist related to
+         * grouping pages by mobility
+         */
+        BUG_ON(page_zone(start_page) != page_zone(end_page));
+#endif
+
+        for (page = start_page; page <= end_page;) {
+                if (!pfn_valid_within(page_to_pfn(page))) {
+                        page++;
+                        continue;
+                }
+
+                if (!PageBuddy(page)) {
+                        page++;
+                        continue;
+                }
+
+                order = page_order(page);
+                list_del(&page->lru);
+                list_add(&page->lru,
+                        &zone->free_area[order].free_list[migratetype]);
+                page += 1 << order;
+                pages_moved += 1 << order;
+        }
+
+        return pages_moved;
+}
+
+int move_freepages_block(struct zone *zone, struct page *page, int migratetype)
+{
+        unsigned long start_pfn, end_pfn;
+        struct page *start_page, *end_page;
+
+        start_pfn = page_to_pfn(page);
+        start_pfn = start_pfn & ~(pageblock_nr_pages-1);
+        start_page = pfn_to_page(start_pfn);
+        end_page = start_page + pageblock_nr_pages - 1;
+        end_pfn = start_pfn + pageblock_nr_pages - 1;
+
+        /* Do not cross zone boundaries */
+        if (start_pfn < zone->zone_start_pfn)
+                start_page = page;
+        if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
+                return 0;
+
+        return move_freepages(zone, start_page, end_page, migratetype);
+}
+
+/* Return the page with the lowest PFN in the list */
+static struct page *min_page(struct list_head *list)
+{
+        unsigned long min_pfn = -1UL;
+        struct page *min_page = NULL, *page;;
+
+        list_for_each_entry(page, list, lru) {
+                unsigned long pfn = page_to_pfn(page);
+                if (pfn < min_pfn) {
+                        min_pfn = pfn;
+                        min_page = page;
+                }
+        }
+
+        return min_page;
+}
+
+/* Remove an element from the buddy allocator from the fallback list */
+static struct page *__rmqueue_fallback(struct zone *zone, int order,
+                                                int start_migratetype)
+{
+        struct free_area * area;
+        int current_order;
+        struct page *page;
+        int migratetype, i;
+
+        /* Find the largest possible block of pages in the other list */
+        for (current_order = MAX_ORDER-1; current_order >= order;
+                                                --current_order) {
+                for (i = 0; i < MIGRATE_TYPES - 1; i++) {
+                        migratetype = fallbacks[start_migratetype][i];
+
+                        /* MIGRATE_RESERVE handled later if necessary */
+                        if (migratetype == MIGRATE_RESERVE)
+                                continue;
+
+                        area = &(zone->free_area[current_order]);
+                        if (list_empty(&area->free_list[migratetype]))
+                                continue;
+
+                        /* Bias kernel allocations towards low pfns */
+                        page = list_entry(area->free_list[migratetype].next,
+                                        struct page, lru);
+                        if (unlikely(start_migratetype != MIGRATE_MOVABLE))
+                                page = min_page(&area->free_list[migratetype]);
+                        area->nr_free--;
+
+                        /*
+                         * If breaking a large block of pages, move all free
+                         * pages to the preferred allocation list. If falling
+                         * back for a reclaimable kernel allocation, be more
+                         * agressive about taking ownership of free pages
+                         */
+                        if (unlikely(current_order >= (pageblock_order >> 1)) ||
+                                        start_migratetype == MIGRATE_RECLAIMABLE) {
+                                unsigned long pages;
+                                pages = move_freepages_block(zone, page,
+                                                                start_migratetype);
+
+                                /* Claim the whole block if over half of it is free */
+                                if (pages >= (1 << (pageblock_order-1)))
+                                        set_pageblock_migratetype(page,
+                                                                start_migratetype);
+
+                                migratetype = start_migratetype;
+                        }
+
+                        /* Remove the page from the freelists */
+                        list_del(&page->lru);
+                        rmv_page_order(page);
+                        __mod_zone_page_state(zone, NR_FREE_PAGES,
+                                                        -(1UL << order));
+
+                        if (current_order == pageblock_order)
+                                set_pageblock_migratetype(page,
+                                                        start_migratetype);
+
+                        expand(zone, page, order, current_order, area, migratetype);
+                        return page;
+                }
+        }
+
+        /* Use MIGRATE_RESERVE rather than fail an allocation */
+        return __rmqueue_smallest(zone, order, MIGRATE_RESERVE);
+}
+
+/*
+ * Do the hard work of removing an element from the buddy allocator.
+ * Call me with the zone->lock already held.
+ */
+static struct page *__rmqueue(struct zone *zone, unsigned int order,
+                                                int migratetype)
+{
+        struct page *page;
+
+        page = __rmqueue_smallest(zone, order, migratetype);
+
+        if (unlikely(!page))
+                page = __rmqueue_fallback(zone, order, migratetype);
+
+        return page;
+}
+
 /* 
  * Obtain a specified number of elements from the buddy allocator, all under
  * a single hold of the lock, for efficiency.  Add them to the supplied list.
  * Returns the number of new pages which were placed at *list.
  */
 static int rmqueue_bulk(struct zone *zone, unsigned int order, 
-                        unsigned long count, struct list_head *list)
+                        unsigned long count, struct list_head *list,
+                        int migratetype)
 {
         int i;
         
         spin_lock(&zone->lock);
         for (i = 0; i < count; ++i) {
-                struct page *page = __rmqueue(zone, order);
+                struct page *page = __rmqueue(zone, order, migratetype);
                 if (unlikely(page == NULL))
                         break;
-                list_add_tail(&page->lru, list);
+                list_add(&page->lru, list);
+                set_page_private(page, migratetype);
         }
         spin_unlock(&zone->lock);
         return i;
@@ -732,7 +930,7 @@ void mark_free_pages(struct zone *zone)
 {
         unsigned long pfn, max_zone_pfn;
         unsigned long flags;
-        int order;
+        int order, t;
         struct list_head *curr;
 
         if (!zone->spanned_pages)
@@ -749,17 +947,18 @@ void mark_free_pages(struct zone *zone)
                                 swsusp_unset_page_free(page);
                 }
 
-        for (order = MAX_ORDER - 1; order >= 0; --order)
-                list_for_each(curr, &zone->free_area[order].free_list) {
+        for_each_migratetype_order(order, t) {
+                list_for_each(curr, &zone->free_area[order].free_list[t]) {
                         unsigned long i;
 
                         pfn = page_to_pfn(list_entry(curr, struct page, lru));
                         for (i = 0; i < (1UL << order); i++)
                                 swsusp_set_page_free(pfn_to_page(pfn + i));
                 }
-
+        }
         spin_unlock_irqrestore(&zone->lock, flags);
 }
+#endif /* CONFIG_PM */
 
 /*
  * Spill all of this CPU's per-cpu pages back into the buddy allocator.
@@ -772,7 +971,25 @@ void drain_local_pages(void)
         __drain_pages(smp_processor_id());
         local_irq_restore(flags);       
 }
-#endif /* CONFIG_HIBERNATION */
+
+void smp_drain_local_pages(void *arg)
+{
+        drain_local_pages();
+}
+
+/*
+ * Spill all the per-cpu pages from all CPUs back into the buddy allocator
+ */
+void drain_all_local_pages(void)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        __drain_pages(smp_processor_id());
+        local_irq_restore(flags);
+
+        smp_call_function(smp_drain_local_pages, NULL, 0, 1);
+}
 
 /*
  * Free a 0-order page
@@ -797,6 +1014,7 @@ static void fastcall free_hot_cold_page(struct page *page, int cold)
         local_irq_save(flags);
         __count_vm_event(PGFREE);
         list_add(&page->lru, &pcp->list);
+        set_page_private(page, get_pageblock_migratetype(page));
         pcp->count++;
         if (pcp->count >= pcp->high) {
                 free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
@@ -846,6 +1064,7 @@ static struct page *buffered_rmqueue(struct zonelist *zonelist,
         struct page *page;
         int cold = !!(gfp_flags & __GFP_COLD);
         int cpu;
+        int migratetype = allocflags_to_migratetype(gfp_flags);
 
 again:
         cpu  = get_cpu();
@@ -856,16 +1075,28 @@ again:
                 local_irq_save(flags);
                 if (!pcp->count) {
                         pcp->count = rmqueue_bulk(zone, 0,
-                                                pcp->batch, &pcp->list);
+                                        pcp->batch, &pcp->list, migratetype);
                         if (unlikely(!pcp->count))
                                 goto failed;
                 }
-                page = list_entry(pcp->list.next, struct page, lru);
+
+                /* Find a page of the appropriate migrate type */
+                list_for_each_entry(page, &pcp->list, lru)
+                        if (page_private(page) == migratetype)
+                                break;
+
+                /* Allocate more to the pcp list if necessary */
+                if (unlikely(&page->lru == &pcp->list)) {
+                        pcp->count += rmqueue_bulk(zone, 0,
+                                        pcp->batch, &pcp->list, migratetype);
+                        page = list_entry(pcp->list.next, struct page, lru);
+                }
+
                 list_del(&page->lru);
                 pcp->count--;
         } else {
                 spin_lock_irqsave(&zone->lock, flags);
-                page = __rmqueue(zone, order);
+                page = __rmqueue(zone, order, migratetype);
                 spin_unlock(&zone->lock);
                 if (!page)
                         goto failed;
@@ -1032,7 +1263,7 @@ int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  *
  * If the zonelist cache is present in the passed in zonelist, then
  * returns a pointer to the allowed node mask (either the current
- * tasks mems_allowed, or node_online_map.)
+ * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  *
  * If the zonelist cache is not available for this zonelist, does
  * nothing and returns NULL.
@@ -1061,7 +1292,7 @@ static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
 
         allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
                                         &cpuset_current_mems_allowed :
-                                        &node_online_map;
+                                        &node_states[N_HIGH_MEMORY];
         return allowednodes;
 }
 
@@ -1183,9 +1414,6 @@ zonelist_scan:
                         !zlc_zone_worth_trying(zonelist, z, allowednodes))
                                 continue;
                 zone = *z;
-                if (unlikely(NUMA_BUILD && (gfp_mask & __GFP_THISNODE) &&
-                        zone->zone_pgdat != zonelist->zones[0]->zone_pgdat))
-                                break;
                 if ((alloc_flags & ALLOC_CPUSET) &&
                         !cpuset_zone_allowed_softwall(zone, gfp_mask))
                                 goto try_next_zone;
@@ -1254,7 +1482,10 @@ restart:
         z = zonelist->zones;  /* the list of zones suitable for gfp_mask */
 
         if (unlikely(*z == NULL)) {
-                /* Should this ever happen?? */
+                /*
+                 * Happens if we have an empty zonelist as a result of
+                 * GFP_THISNODE being used on a memoryless node
+                 */
                 return NULL;
         }
 
@@ -1346,6 +1577,9 @@ nofail_alloc:
 
         cond_resched();
 
+        if (order != 0)
+                drain_all_local_pages();
+
         if (likely(did_some_progress)) {
                 page = get_page_from_freelist(gfp_mask, order,
                                                 zonelist, alloc_flags);
@@ -1794,7 +2028,7 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
                 return node;
         }
 
-        for_each_online_node(n) {
+        for_each_node_state(n, N_HIGH_MEMORY) {
                 cpumask_t tmp;
 
                 /* Don't want a node to appear more than once */
@@ -1850,6 +2084,22 @@ static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
 }
 
 /*
+ * Build gfp_thisnode zonelists
+ */
+static void build_thisnode_zonelists(pg_data_t *pgdat)
+{
+        enum zone_type i;
+        int j;
+        struct zonelist *zonelist;
+
+        for (i = 0; i < MAX_NR_ZONES; i++) {
+                zonelist = pgdat->node_zonelists + MAX_NR_ZONES + i;
+                j = build_zonelists_node(pgdat, zonelist, 0, i);
+                zonelist->zones[j] = NULL;
+        }
+}
+
+/*
  * Build zonelists ordered by zone and nodes within zones.
  * This results in conserving DMA zone[s] until all Normal memory is
  * exhausted, but results in overflowing to remote node while memory
@@ -1915,7 +2165,8 @@ static int default_zonelist_order(void)
          * If there is a node whose DMA/DMA32 memory is very big area on
          * local memory, NODE_ORDER may be suitable.
          */
-        average_size = total_size / (num_online_nodes() + 1);
+        average_size = total_size /
+                                (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
         for_each_online_node(nid) {
                 low_kmem_size = 0;
                 total_size = 0;
@@ -1953,7 +2204,7 @@ static void build_zonelists(pg_data_t *pgdat)
         int order = current_zonelist_order;
 
         /* initialize zonelists */
-        for (i = 0; i < MAX_NR_ZONES; i++) {
+        for (i = 0; i < MAX_ZONELISTS; i++) {
                 zonelist = pgdat->node_zonelists + i;
                 zonelist->zones[0] = NULL;
         }
@@ -1998,6 +2249,8 @@ static void build_zonelists(pg_data_t *pgdat)
                 /* calculate node order -- i.e., DMA last! */
                 build_zonelists_in_zone_order(pgdat, j);
         }
+
+        build_thisnode_zonelists(pgdat);
 }
 
 /* Construct the zonelist performance cache - see further mmzone.h */
@@ -2078,8 +2331,10 @@ static int __build_all_zonelists(void *dummy)
         int nid;
 
         for_each_online_node(nid) {
-                build_zonelists(NODE_DATA(nid));
-                build_zonelist_cache(NODE_DATA(nid));
+                pg_data_t *pgdat = NODE_DATA(nid);
+
+                build_zonelists(pgdat);
+                build_zonelist_cache(pgdat);
         }
         return 0;
 }
@@ -2098,9 +2353,23 @@ void build_all_zonelists(void)
                 /* cpuset refresh routine should be here */
         }
         vm_total_pages = nr_free_pagecache_pages();
-        printk("Built %i zonelists in %s order.  Total pages: %ld\n",
+        /*
+         * Disable grouping by mobility if the number of pages in the
+         * system is too low to allow the mechanism to work. It would be
+         * more accurate, but expensive to check per-zone. This check is
+         * made on memory-hotadd so a system can start with mobility
+         * disabled and enable it later
+         */
+        if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
+                page_group_by_mobility_disabled = 1;
+        else
+                page_group_by_mobility_disabled = 0;
+
+        printk("Built %i zonelists in %s order, mobility grouping %s.  "
+                "Total pages: %ld\n",
                         num_online_nodes(),
                         zonelist_order_name[current_zonelist_order],
+                        page_group_by_mobility_disabled ? "off" : "on",
                         vm_total_pages);
 #ifdef CONFIG_NUMA
         printk("Policy zone: %s\n", zone_names[policy_zone]);
@@ -2176,6 +2445,61 @@ static inline unsigned long wait_table_bits(unsigned long size)
 #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
 
 /*
+ * Mark a number of pageblocks as MIGRATE_RESERVE. The number
+ * of blocks reserved is based on zone->pages_min. The memory within the
+ * reserve will tend to store contiguous free pages. Setting min_free_kbytes
+ * higher will lead to a bigger reserve which will get freed as contiguous
+ * blocks as reclaim kicks in
+ */
+static void setup_zone_migrate_reserve(struct zone *zone)
+{
+        unsigned long start_pfn, pfn, end_pfn;
+        struct page *page;
+        unsigned long reserve, block_migratetype;
+
+        /* Get the start pfn, end pfn and the number of blocks to reserve */
+        start_pfn = zone->zone_start_pfn;
+        end_pfn = start_pfn + zone->spanned_pages;
+        reserve = roundup(zone->pages_min, pageblock_nr_pages) >>
+                                                        pageblock_order;
+
+        for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
+                if (!pfn_valid(pfn))
+                        continue;
+                page = pfn_to_page(pfn);
+
+                /* Blocks with reserved pages will never free, skip them. */
+                if (PageReserved(page))
+                        continue;
+
+                block_migratetype = get_pageblock_migratetype(page);
+
+                /* If this block is reserved, account for it */
+                if (reserve > 0 && block_migratetype == MIGRATE_RESERVE) {
+                        reserve--;
+                        continue;
+                }
+
+                /* Suitable for reserving if this block is movable */
+                if (reserve > 0 && block_migratetype == MIGRATE_MOVABLE) {
+                        set_pageblock_migratetype(page, MIGRATE_RESERVE);
+                        move_freepages_block(zone, page, MIGRATE_RESERVE);
+                        reserve--;
+                        continue;
+                }
+
+                /*
+                 * If the reserve is met and this is a previous reserved block,
+                 * take it back
+                 */
+                if (block_migratetype == MIGRATE_RESERVE) {
+                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+                        move_freepages_block(zone, page, MIGRATE_MOVABLE);
+                }
+        }
+}
+
+/*
  * Initially all pages are reserved - free ones are freed
  * up by free_all_bootmem() once the early boot process is
  * done. Non-atomic initialization, single-pass.
@@ -2204,6 +2528,19 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
                 init_page_count(page);
                 reset_page_mapcount(page);
                 SetPageReserved(page);
+
+                /*
+                 * Mark the block movable so that blocks are reserved for
+                 * movable at startup. This will force kernel allocations
+                 * to reserve their blocks rather than leaking throughout
+                 * the address space during boot when many long-lived
+                 * kernel allocations are made. Later some blocks near
+                 * the start are marked MIGRATE_RESERVE by
+                 * setup_zone_migrate_reserve()
+                 */
+                if ((pfn & (pageblock_nr_pages-1)))
+                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+
                 INIT_LIST_HEAD(&page->lru);
 #ifdef WANT_PAGE_VIRTUAL
                 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
@@ -2216,9 +2553,9 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
 static void __meminit zone_init_free_lists(struct pglist_data *pgdat,
                                 struct zone *zone, unsigned long size)
 {
-        int order;
-        for (order = 0; order < MAX_ORDER ; order++) {
-                INIT_LIST_HEAD(&zone->free_area[order].free_list);
+        int order, t;
+        for_each_migratetype_order(order, t) {
+                INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
                 zone->free_area[order].nr_free = 0;
         }
 }
@@ -2324,6 +2661,9 @@ static struct per_cpu_pageset boot_pageset[NR_CPUS];
 static int __cpuinit process_zones(int cpu)
 {
         struct zone *zone, *dzone;
+        int node = cpu_to_node(cpu);
+
+        node_set_state(node, N_CPU);    /* this node has a cpu */
 
         for_each_zone(zone) {
 
@@ -2331,7 +2671,7 @@ static int __cpuinit process_zones(int cpu)
                         continue;
 
                 zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset),
-                                         GFP_KERNEL, cpu_to_node(cpu));
+                                         GFP_KERNEL, node);
                 if (!zone_pcp(zone, cpu))
                         goto bad;
 
@@ -2444,7 +2784,7 @@ int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
                  * To use this new node's memory, further consideration will be
                  * necessary.
                  */
-                zone->wait_table = (wait_queue_head_t *)vmalloc(alloc_size);
+                zone->wait_table = vmalloc(alloc_size);
         }
         if (!zone->wait_table)
                 return -ENOMEM;
@@ -2680,10 +3020,8 @@ void __meminit get_pfn_range_for_nid(unsigned int nid,
                 *end_pfn = max(*end_pfn, early_node_map[i].end_pfn);
         }
 
-        if (*start_pfn == -1UL) {
-                printk(KERN_WARNING "Node %u active with no memory\n", nid);
+        if (*start_pfn == -1UL)
                 *start_pfn = 0;
-        }
 
         /* Push the node boundaries out if requested */
         account_node_boundary(nid, start_pfn, end_pfn);
@@ -2901,6 +3239,62 @@ static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
                                                         realtotalpages);
 }
 
+#ifndef CONFIG_SPARSEMEM
+/*
+ * Calculate the size of the zone->blockflags rounded to an unsigned long
+ * Start by making sure zonesize is a multiple of pageblock_order by rounding
+ * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
+ * round what is now in bits to nearest long in bits, then return it in
+ * bytes.
+ */
+static unsigned long __init usemap_size(unsigned long zonesize)
+{
+        unsigned long usemapsize;
+
+        usemapsize = roundup(zonesize, pageblock_nr_pages);
+        usemapsize = usemapsize >> pageblock_order;
+        usemapsize *= NR_PAGEBLOCK_BITS;
+        usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
+
+        return usemapsize / 8;
+}
+
+static void __init setup_usemap(struct pglist_data *pgdat,
+                                struct zone *zone, unsigned long zonesize)
+{
+        unsigned long usemapsize = usemap_size(zonesize);
+        zone->pageblock_flags = NULL;
+        if (usemapsize) {
+                zone->pageblock_flags = alloc_bootmem_node(pgdat, usemapsize);
+                memset(zone->pageblock_flags, 0, usemapsize);
+        }
+}
+#else
+static void inline setup_usemap(struct pglist_data *pgdat,
+                                struct zone *zone, unsigned long zonesize) {}
+#endif /* CONFIG_SPARSEMEM */
+
+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
+/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
+static inline void __init set_pageblock_order(unsigned int order)
+{
+        /* Check that pageblock_nr_pages has not already been setup */
+        if (pageblock_order)
+                return;
+
+        /*
+         * Assume the largest contiguous order of interest is a huge page.
+         * This value may be variable depending on boot parameters on IA64
+         */
+        pageblock_order = order;
+}
+#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
+
+/* Defined this way to avoid accidently referencing HUGETLB_PAGE_ORDER */
+#define set_pageblock_order(x)  do {} while (0)
+
+#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
+
 /*
  * Set up the zone data structures:
  *   - mark all pages reserved
@@ -2981,6 +3375,8 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat,
                 if (!size)
                         continue;
 
+                set_pageblock_order(HUGETLB_PAGE_ORDER);
+                setup_usemap(pgdat, zone, size);
                 ret = init_currently_empty_zone(zone, zone_start_pfn,
                                                 size, MEMMAP_EARLY);
                 BUG_ON(ret);
@@ -3234,16 +3630,24 @@ unsigned long __init find_max_pfn_with_active_regions(void)
         return max_pfn;
 }
 
-unsigned long __init early_calculate_totalpages(void)
+/*
+ * early_calculate_totalpages()
+ * Sum pages in active regions for movable zone.
+ * Populate N_HIGH_MEMORY for calculating usable_nodes.
+ */
+static unsigned long __init early_calculate_totalpages(void)
 {
         int i;
         unsigned long totalpages = 0;
 
-        for (i = 0; i < nr_nodemap_entries; i++)
-                totalpages += early_node_map[i].end_pfn -
+        for (i = 0; i < nr_nodemap_entries; i++) {
+                unsigned long pages = early_node_map[i].end_pfn -
                                                 early_node_map[i].start_pfn;
-
-        return totalpages;
+                totalpages += pages;
+                if (pages)
+                        node_set_state(early_node_map[i].nid, N_HIGH_MEMORY);
+        }
+        return totalpages;
 }
 
 /*
@@ -3257,7 +3661,8 @@ void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
         int i, nid;
         unsigned long usable_startpfn;
         unsigned long kernelcore_node, kernelcore_remaining;
-        int usable_nodes = num_online_nodes();
+        unsigned long totalpages = early_calculate_totalpages();
+        int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
 
         /*
          * If movablecore was specified, calculate what size of
@@ -3268,7 +3673,6 @@ void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
          * what movablecore would have allowed.
          */
         if (required_movablecore) {
-                unsigned long totalpages = early_calculate_totalpages();
                 unsigned long corepages;
 
                 /*
@@ -3293,7 +3697,7 @@ void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
 restart:
         /* Spread kernelcore memory as evenly as possible throughout nodes */
         kernelcore_node = required_kernelcore / usable_nodes;
-        for_each_online_node(nid) {
+        for_each_node_state(nid, N_HIGH_MEMORY) {
                 /*
                  * Recalculate kernelcore_node if the division per node
                  * now exceeds what is necessary to satisfy the requested
@@ -3385,6 +3789,20 @@ restart:
                         roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
 }
 
+/* Any regular memory on that node ? */
+static void check_for_regular_memory(pg_data_t *pgdat)
+{
+#ifdef CONFIG_HIGHMEM
+        enum zone_type zone_type;
+
+        for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
+                struct zone *zone = &pgdat->node_zones[zone_type];
+                if (zone->present_pages)
+                        node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
+        }
+#endif
+}
+
 /**
  * free_area_init_nodes - Initialise all pg_data_t and zone data
  * @max_zone_pfn: an array of max PFNs for each zone
@@ -3459,6 +3877,11 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
                 pg_data_t *pgdat = NODE_DATA(nid);
                 free_area_init_node(nid, pgdat, NULL,
                                 find_min_pfn_for_node(nid), NULL);
+
+                /* Any memory on that node */
+                if (pgdat->node_present_pages)
+                        node_set_state(nid, N_HIGH_MEMORY);
+                check_for_regular_memory(pgdat);
         }
 }
 
@@ -3673,6 +4096,7 @@ void setup_per_zone_pages_min(void)
 
                 zone->pages_low   = zone->pages_min + (tmp >> 2);
                 zone->pages_high  = zone->pages_min + (tmp >> 1);
+                setup_zone_migrate_reserve(zone);
                 spin_unlock_irqrestore(&zone->lru_lock, flags);
         }
 
@@ -3934,4 +4358,169 @@ EXPORT_SYMBOL(pfn_to_page);
 EXPORT_SYMBOL(page_to_pfn);
 #endif /* CONFIG_OUT_OF_LINE_PFN_TO_PAGE */
 
+/* Return a pointer to the bitmap storing bits affecting a block of pages */
+static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
+                                                        unsigned long pfn)
+{
+#ifdef CONFIG_SPARSEMEM
+        return __pfn_to_section(pfn)->pageblock_flags;
+#else
+        return zone->pageblock_flags;
+#endif /* CONFIG_SPARSEMEM */
+}
+
+static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
+{
+#ifdef CONFIG_SPARSEMEM
+        pfn &= (PAGES_PER_SECTION-1);
+        return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
+#else
+        pfn = pfn - zone->zone_start_pfn;
+        return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
+#endif /* CONFIG_SPARSEMEM */
+}
+
+/**
+ * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
+ * @page: The page within the block of interest
+ * @start_bitidx: The first bit of interest to retrieve
+ * @end_bitidx: The last bit of interest
+ * returns pageblock_bits flags
+ */
+unsigned long get_pageblock_flags_group(struct page *page,
+                                        int start_bitidx, int end_bitidx)
+{
+        struct zone *zone;
+        unsigned long *bitmap;
+        unsigned long pfn, bitidx;
+        unsigned long flags = 0;
+        unsigned long value = 1;
+
+        zone = page_zone(page);
+        pfn = page_to_pfn(page);
+        bitmap = get_pageblock_bitmap(zone, pfn);
+        bitidx = pfn_to_bitidx(zone, pfn);
+
+        for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
+                if (test_bit(bitidx + start_bitidx, bitmap))
+                        flags |= value;
+
+        return flags;
+}
 
+/**
+ * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
+ * @page: The page within the block of interest
+ * @start_bitidx: The first bit of interest
+ * @end_bitidx: The last bit of interest
+ * @flags: The flags to set
+ */
+void set_pageblock_flags_group(struct page *page, unsigned long flags,
+                                        int start_bitidx, int end_bitidx)
+{
+        struct zone *zone;
+        unsigned long *bitmap;
+        unsigned long pfn, bitidx;
+        unsigned long value = 1;
+
+        zone = page_zone(page);
+        pfn = page_to_pfn(page);
+        bitmap = get_pageblock_bitmap(zone, pfn);
+        bitidx = pfn_to_bitidx(zone, pfn);
+
+        for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
+                if (flags & value)
+                        __set_bit(bitidx + start_bitidx, bitmap);
+                else
+                        __clear_bit(bitidx + start_bitidx, bitmap);
+}
+
+/*
+ * This is designed as sub function...plz see page_isolation.c also.
+ * set/clear page block's type to be ISOLATE.
+ * page allocater never alloc memory from ISOLATE block.
+ */
+
+int set_migratetype_isolate(struct page *page)
+{
+        struct zone *zone;
+        unsigned long flags;
+        int ret = -EBUSY;
+
+        zone = page_zone(page);
+        spin_lock_irqsave(&zone->lock, flags);
+        /*
+         * In future, more migrate types will be able to be isolation target.
+         */
+        if (get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
+                goto out;
+        set_pageblock_migratetype(page, MIGRATE_ISOLATE);
+        move_freepages_block(zone, page, MIGRATE_ISOLATE);
+        ret = 0;
+out:
+        spin_unlock_irqrestore(&zone->lock, flags);
+        if (!ret)
+                drain_all_local_pages();
+        return ret;
+}
+
+void unset_migratetype_isolate(struct page *page)
+{
+        struct zone *zone;
+        unsigned long flags;
+        zone = page_zone(page);
+        spin_lock_irqsave(&zone->lock, flags);
+        if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
+                goto out;
+        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+        move_freepages_block(zone, page, MIGRATE_MOVABLE);
+out:
+        spin_unlock_irqrestore(&zone->lock, flags);
+}
+
+#ifdef CONFIG_MEMORY_HOTREMOVE
+/*
+ * All pages in the range must be isolated before calling this.
+ */
+void
+__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
+{
+        struct page *page;
+        struct zone *zone;
+        int order, i;
+        unsigned long pfn;
+        unsigned long flags;
+        /* find the first valid pfn */
+        for (pfn = start_pfn; pfn < end_pfn; pfn++)
+                if (pfn_valid(pfn))
+                        break;
+        if (pfn == end_pfn)
+                return;
+        zone = page_zone(pfn_to_page(pfn));
+        spin_lock_irqsave(&zone->lock, flags);
+        pfn = start_pfn;
+        while (pfn < end_pfn) {
+                if (!pfn_valid(pfn)) {
+                        pfn++;
+                        continue;
+                }
+                page = pfn_to_page(pfn);
+                BUG_ON(page_count(page));
+                BUG_ON(!PageBuddy(page));
+                order = page_order(page);
+#ifdef CONFIG_DEBUG_VM
+                printk(KERN_INFO "remove from free list %lx %d %lx\n",
+                       pfn, 1 << order, end_pfn);
+#endif
+                list_del(&page->lru);
+                rmv_page_order(page);
+                zone->free_area[order].nr_free--;
+                __mod_zone_page_state(zone, NR_FREE_PAGES,
+                                      - (1UL << order));
+                for (i = 0; i < (1 << order); i++)
+                        SetPageReserved((page+i));
+                pfn += (1 << order);
+        }
+        spin_unlock_irqrestore(&zone->lock, flags);
+}
+#endif