1 files changed, 350 insertions, 177 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2bc2ac63f41e..d03c946d5566 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -48,7 +48,9 @@
 #include <linux/page_cgroup.h>
 #include <linux/debugobjects.h>
 #include <linux/kmemleak.h>
+#include <linux/memory.h>
 #include <trace/events/kmem.h>
+#include <linux/ftrace_event.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>
@@ -75,6 +77,31 @@ unsigned long totalreserve_pages __read_mostly;
 int percpu_pagelist_fraction;
 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 
+#ifdef CONFIG_PM_SLEEP
+/*
+ * The following functions are used by the suspend/hibernate code to temporarily
+ * change gfp_allowed_mask in order to avoid using I/O during memory allocations
+ * while devices are suspended.  To avoid races with the suspend/hibernate code,
+ * they should always be called with pm_mutex held (gfp_allowed_mask also should
+ * only be modified with pm_mutex held, unless the suspend/hibernate code is
+ * guaranteed not to run in parallel with that modification).
+ */
+void set_gfp_allowed_mask(gfp_t mask)
+{
+        WARN_ON(!mutex_is_locked(&pm_mutex));
+        gfp_allowed_mask = mask;
+}
+
+gfp_t clear_gfp_allowed_mask(gfp_t mask)
+{
+        gfp_t ret = gfp_allowed_mask;
+
+        WARN_ON(!mutex_is_locked(&pm_mutex));
+        gfp_allowed_mask &= ~mask;
+        return ret;
+}
+#endif /* CONFIG_PM_SLEEP */
+
 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 int pageblock_order __read_mostly;
 #endif
@@ -262,10 +289,7 @@ static void bad_page(struct page *page)
 
         printk(KERN_ALERT "BUG: Bad page state in process %s  pfn:%05lx\n",
                 current->comm, page_to_pfn(page));
-        printk(KERN_ALERT
-                "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n",
-                page, (void *)page->flags, page_count(page),
-                page_mapcount(page), page->mapping, page->index);
+        dump_page(page);
 
         dump_stack();
 out:
@@ -486,7 +510,6 @@ static inline void __free_one_page(struct page *page,
         zone->free_area[order].nr_free++;
 }
 
-#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
 /*
  * free_page_mlock() -- clean up attempts to free and mlocked() page.
  * Page should not be on lru, so no need to fix that up.
@@ -497,9 +520,6 @@ static inline void free_page_mlock(struct page *page)
         __dec_zone_page_state(page, NR_MLOCK);
         __count_vm_event(UNEVICTABLE_MLOCKFREED);
 }
-#else
-static void free_page_mlock(struct page *page) { }
-#endif
 
 static inline int free_pages_check(struct page *page)
 {
@@ -533,7 +553,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
         int batch_free = 0;
 
         spin_lock(&zone->lock);
-        zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
+        zone->all_unreclaimable = 0;
         zone->pages_scanned = 0;
 
         __mod_zone_page_state(zone, NR_FREE_PAGES, count);
@@ -559,8 +579,9 @@ static void free_pcppages_bulk(struct zone *zone, int count,
                         page = list_entry(list->prev, struct page, lru);
                         /* must delete as __free_one_page list manipulates */
                         list_del(&page->lru);
-                        __free_one_page(page, zone, 0, migratetype);
-                        trace_mm_page_pcpu_drain(page, 0, migratetype);
+                        /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
+                        __free_one_page(page, zone, 0, page_private(page));
+                        trace_mm_page_pcpu_drain(page, 0, page_private(page));
                 } while (--count && --batch_free && !list_empty(list));
         }
         spin_unlock(&zone->lock);
@@ -570,7 +591,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
                                 int migratetype)
 {
         spin_lock(&zone->lock);
-        zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
+        zone->all_unreclaimable = 0;
         zone->pages_scanned = 0;
 
         __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
@@ -585,6 +606,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
         int bad = 0;
         int wasMlocked = __TestClearPageMlocked(page);
 
+        trace_mm_page_free_direct(page, order);
         kmemcheck_free_shadow(page, order);
 
         for (i = 0 ; i < (1 << order) ; ++i)
@@ -1011,10 +1033,10 @@ static void drain_pages(unsigned int cpu)
                 struct per_cpu_pageset *pset;
                 struct per_cpu_pages *pcp;
 
-                pset = zone_pcp(zone, cpu);
+                local_irq_save(flags);
+                pset = per_cpu_ptr(zone->pageset, cpu);
 
                 pcp = &pset->pcp;
-                local_irq_save(flags);
                 free_pcppages_bulk(zone, pcp->count, pcp);
                 pcp->count = 0;
                 local_irq_restore(flags);
@@ -1075,8 +1097,9 @@ void mark_free_pages(struct zone *zone)
 
 /*
  * Free a 0-order page
+ * cold == 1 ? free a cold page : free a hot page
  */
-static void free_hot_cold_page(struct page *page, int cold)
+void free_hot_cold_page(struct page *page, int cold)
 {
         struct zone *zone = page_zone(page);
         struct per_cpu_pages *pcp;
@@ -1084,6 +1107,7 @@ static void free_hot_cold_page(struct page *page, int cold)
         int migratetype;
         int wasMlocked = __TestClearPageMlocked(page);
 
+        trace_mm_page_free_direct(page, 0);
         kmemcheck_free_shadow(page, 0);
 
         if (PageAnon(page))
@@ -1098,7 +1122,6 @@ static void free_hot_cold_page(struct page *page, int cold)
         arch_free_page(page, 0);
         kernel_map_pages(page, 1, 0);
 
-        pcp = &zone_pcp(zone, get_cpu())->pcp;
         migratetype = get_pageblock_migratetype(page);
         set_page_private(page, migratetype);
         local_irq_save(flags);
@@ -1121,6 +1144,7 @@ static void free_hot_cold_page(struct page *page, int cold)
                 migratetype = MIGRATE_MOVABLE;
         }
 
+        pcp = &this_cpu_ptr(zone->pageset)->pcp;
         if (cold)
                 list_add_tail(&page->lru, &pcp->lists[migratetype]);
         else
@@ -1133,15 +1157,8 @@ static void free_hot_cold_page(struct page *page, int cold)
 
 out:
         local_irq_restore(flags);
-        put_cpu();
 }
 
-void free_hot_page(struct page *page)
-{
-        trace_mm_page_free_direct(page, 0);
-        free_hot_cold_page(page, 0);
-}
-        
 /*
  * split_page takes a non-compound higher-order page, and splits it into
  * n (1<<order) sub-pages: page[0..n]
@@ -1183,17 +1200,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
         unsigned long flags;
         struct page *page;
         int cold = !!(gfp_flags & __GFP_COLD);
-        int cpu;
 
 again:
-        cpu  = get_cpu();
         if (likely(order == 0)) {
                 struct per_cpu_pages *pcp;
                 struct list_head *list;
 
-                pcp = &zone_pcp(zone, cpu)->pcp;
-                list = &pcp->lists[migratetype];
                 local_irq_save(flags);
+                pcp = &this_cpu_ptr(zone->pageset)->pcp;
+                list = &pcp->lists[migratetype];
                 if (list_empty(list)) {
                         pcp->count += rmqueue_bulk(zone, 0,
                                         pcp->batch, list,
@@ -1225,16 +1240,15 @@ again:
                 }
                 spin_lock_irqsave(&zone->lock, flags);
                 page = __rmqueue(zone, order, migratetype);
-                __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
                 spin_unlock(&zone->lock);
                 if (!page)
                         goto failed;
+                __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
         }
 
         __count_zone_vm_events(PGALLOC, zone, 1 << order);
         zone_statistics(preferred_zone, zone);
         local_irq_restore(flags);
-        put_cpu();
 
         VM_BUG_ON(bad_range(zone, page));
         if (prep_new_page(page, order, gfp_flags))
@@ -1243,7 +1257,6 @@ again:
 
 failed:
         local_irq_restore(flags);
-        put_cpu();
         return NULL;
 }
 
@@ -1658,12 +1671,22 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
         if (page)
                 goto out;
 
-        /* The OOM killer will not help higher order allocs */
-        if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_NOFAIL))
-                goto out;
-
+        if (!(gfp_mask & __GFP_NOFAIL)) {
+                /* The OOM killer will not help higher order allocs */
+                if (order > PAGE_ALLOC_COSTLY_ORDER)
+                        goto out;
+                /*
+                 * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
+                 * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
+                 * The caller should handle page allocation failure by itself if
+                 * it specifies __GFP_THISNODE.
+                 * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
+                 */
+                if (gfp_mask & __GFP_THISNODE)
+                        goto out;
+        }
         /* Exhausted what can be done so it's blamo time */
-        out_of_memory(zonelist, gfp_mask, order);
+        out_of_memory(zonelist, gfp_mask, order, nodemask);
 
 out:
         clear_zonelist_oom(zonelist, gfp_mask);
@@ -2005,9 +2028,8 @@ void __pagevec_free(struct pagevec *pvec)
 void __free_pages(struct page *page, unsigned int order)
 {
         if (put_page_testzero(page)) {
-                trace_mm_page_free_direct(page, order);
                 if (order == 0)
-                        free_hot_page(page);
+                        free_hot_cold_page(page, 0);
                 else
                         __free_pages_ok(page, order);
         }
@@ -2172,7 +2194,7 @@ void show_free_areas(void)
                 for_each_online_cpu(cpu) {
                         struct per_cpu_pageset *pageset;
 
-                        pageset = zone_pcp(zone, cpu);
+                        pageset = per_cpu_ptr(zone->pageset, cpu);
 
                         printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
                                cpu, pageset->pcp.high,
@@ -2263,7 +2285,7 @@ void show_free_areas(void)
                         K(zone_page_state(zone, NR_BOUNCE)),
                         K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
                         zone->pages_scanned,
-                        (zone_is_all_unreclaimable(zone) ? "yes" : "no")
+                        (zone->all_unreclaimable ? "yes" : "no")
                         );
                 printk("lowmem_reserve[]:");
                 for (i = 0; i < MAX_NR_ZONES; i++)
@@ -2395,13 +2417,14 @@ int numa_zonelist_order_handler(ctl_table *table, int write,
 {
         char saved_string[NUMA_ZONELIST_ORDER_LEN];
         int ret;
+        static DEFINE_MUTEX(zl_order_mutex);
 
+        mutex_lock(&zl_order_mutex);
         if (write)
-                strncpy(saved_string, (char*)table->data,
-                        NUMA_ZONELIST_ORDER_LEN);
+                strcpy(saved_string, (char*)table->data);
         ret = proc_dostring(table, write, buffer, length, ppos);
         if (ret)
-                return ret;
+                goto out;
         if (write) {
                 int oldval = user_zonelist_order;
                 if (__parse_numa_zonelist_order((char*)table->data)) {
@@ -2414,7 +2437,9 @@ int numa_zonelist_order_handler(ctl_table *table, int write,
                 } else if (oldval != user_zonelist_order)
                         build_all_zonelists();
         }
-        return 0;
+out:
+        mutex_unlock(&zl_order_mutex);
+        return ret;
 }
 
 
@@ -2734,10 +2759,29 @@ static void build_zonelist_cache(pg_data_t *pgdat)
 
 #endif  /* CONFIG_NUMA */
 
+/*
+ * Boot pageset table. One per cpu which is going to be used for all
+ * zones and all nodes. The parameters will be set in such a way
+ * that an item put on a list will immediately be handed over to
+ * the buddy list. This is safe since pageset manipulation is done
+ * with interrupts disabled.
+ *
+ * The boot_pagesets must be kept even after bootup is complete for
+ * unused processors and/or zones. They do play a role for bootstrapping
+ * hotplugged processors.
+ *
+ * zoneinfo_show() and maybe other functions do
+ * not check if the processor is online before following the pageset pointer.
+ * Other parts of the kernel may not check if the zone is available.
+ */
+static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
+static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
+
 /* return values int ....just for stop_machine() */
 static int __build_all_zonelists(void *dummy)
 {
         int nid;
+        int cpu;
 
 #ifdef CONFIG_NUMA
         memset(node_load, 0, sizeof(node_load));
@@ -2748,6 +2792,23 @@ static int __build_all_zonelists(void *dummy)
                 build_zonelists(pgdat);
                 build_zonelist_cache(pgdat);
         }
+
+        /*
+         * Initialize the boot_pagesets that are going to be used
+         * for bootstrapping processors. The real pagesets for
+         * each zone will be allocated later when the per cpu
+         * allocator is available.
+         *
+         * boot_pagesets are used also for bootstrapping offline
+         * cpus if the system is already booted because the pagesets
+         * are needed to initialize allocators on a specific cpu too.
+         * F.e. the percpu allocator needs the page allocator which
+         * needs the percpu allocator in order to allocate its pagesets
+         * (a chicken-egg dilemma).
+         */
+        for_each_possible_cpu(cpu)
+                setup_pageset(&per_cpu(boot_pageset, cpu), 0);
+
         return 0;
 }
 
@@ -3085,121 +3146,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p,
                 pcp->batch = PAGE_SHIFT * 8;
 }
 
-
-#ifdef CONFIG_NUMA
-/*
- * Boot pageset table. One per cpu which is going to be used for all
- * zones and all nodes. The parameters will be set in such a way
- * that an item put on a list will immediately be handed over to
- * the buddy list. This is safe since pageset manipulation is done
- * with interrupts disabled.
- *
- * Some NUMA counter updates may also be caught by the boot pagesets.
- *
- * The boot_pagesets must be kept even after bootup is complete for
- * unused processors and/or zones. They do play a role for bootstrapping
- * hotplugged processors.
- *
- * zoneinfo_show() and maybe other functions do
- * not check if the processor is online before following the pageset pointer.
- * Other parts of the kernel may not check if the zone is available.
- */
-static struct per_cpu_pageset boot_pageset[NR_CPUS];
-
 /*
- * Dynamically allocate memory for the
- * per cpu pageset array in struct zone.
+ * Allocate per cpu pagesets and initialize them.
+ * Before this call only boot pagesets were available.
+ * Boot pagesets will no longer be used by this processorr
+ * after setup_per_cpu_pageset().
  */
-static int __cpuinit process_zones(int cpu)
+void __init setup_per_cpu_pageset(void)
 {
-        struct zone *zone, *dzone;
-        int node = cpu_to_node(cpu);
-
-        node_set_state(node, N_CPU);    /* this node has a cpu */
+        struct zone *zone;
+        int cpu;
 
         for_each_populated_zone(zone) {
-                zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset),
-                                         GFP_KERNEL, node);
-                if (!zone_pcp(zone, cpu))
-                        goto bad;
+                zone->pageset = alloc_percpu(struct per_cpu_pageset);
 
-                setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone));
+                for_each_possible_cpu(cpu) {
+                        struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
 
-                if (percpu_pagelist_fraction)
-                        setup_pagelist_highmark(zone_pcp(zone, cpu),
-                                (zone->present_pages / percpu_pagelist_fraction));
-        }
+                        setup_pageset(pcp, zone_batchsize(zone));
 
-        return 0;
-bad:
-        for_each_zone(dzone) {
-                if (!populated_zone(dzone))
-                        continue;
-                if (dzone == zone)
-                        break;
-                kfree(zone_pcp(dzone, cpu));
-                zone_pcp(dzone, cpu) = &boot_pageset[cpu];
-        }
-        return -ENOMEM;
-}
-
-static inline void free_zone_pagesets(int cpu)
-{
-        struct zone *zone;
-
-        for_each_zone(zone) {
-                struct per_cpu_pageset *pset = zone_pcp(zone, cpu);
-
-                /* Free per_cpu_pageset if it is slab allocated */
-                if (pset != &boot_pageset[cpu])
-                        kfree(pset);
-                zone_pcp(zone, cpu) = &boot_pageset[cpu];
-        }
-}
-
-static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
-                unsigned long action,
-                void *hcpu)
-{
-        int cpu = (long)hcpu;
-        int ret = NOTIFY_OK;
-
-        switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
-                if (process_zones(cpu))
-                        ret = NOTIFY_BAD;
-                break;
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
-        case CPU_DEAD:
-        case CPU_DEAD_FROZEN:
-                free_zone_pagesets(cpu);
-                break;
-        default:
-                break;
+                        if (percpu_pagelist_fraction)
+                                setup_pagelist_highmark(pcp,
+                                        (zone->present_pages /
+                                                percpu_pagelist_fraction));
+                }
         }
-        return ret;
 }
 
-static struct notifier_block __cpuinitdata pageset_notifier =
-        { &pageset_cpuup_callback, NULL, 0 };
-
-void __init setup_per_cpu_pageset(void)
-{
-        int err;
-
-        /* Initialize per_cpu_pageset for cpu 0.
-         * A cpuup callback will do this for every cpu
-         * as it comes online
-         */
-        err = process_zones(smp_processor_id());
-        BUG_ON(err);
-        register_cpu_notifier(&pageset_notifier);
-}
-
-#endif
-
 static noinline __init_refok
 int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
 {
@@ -3249,11 +3222,11 @@ static int __zone_pcp_update(void *data)
         int cpu;
         unsigned long batch = zone_batchsize(zone), flags;
 
-        for (cpu = 0; cpu < NR_CPUS; cpu++) {
+        for_each_possible_cpu(cpu) {
                 struct per_cpu_pageset *pset;
                 struct per_cpu_pages *pcp;
 
-                pset = zone_pcp(zone, cpu);
+                pset = per_cpu_ptr(zone->pageset, cpu);
                 pcp = &pset->pcp;
 
                 local_irq_save(flags);
@@ -3271,21 +3244,17 @@ void zone_pcp_update(struct zone *zone)
 
 static __meminit void zone_pcp_init(struct zone *zone)
 {
-        int cpu;
-        unsigned long batch = zone_batchsize(zone);
+        /*
+         * per cpu subsystem is not up at this point. The following code
+         * relies on the ability of the linker to provide the
+         * offset of a (static) per cpu variable into the per cpu area.
+         */
+        zone->pageset = &boot_pageset;
 
-        for (cpu = 0; cpu < NR_CPUS; cpu++) {
-#ifdef CONFIG_NUMA
-                /* Early boot. Slab allocator not functional yet */
-                zone_pcp(zone, cpu) = &boot_pageset[cpu];
-                setup_pageset(&boot_pageset[cpu],0);
-#else
-                setup_pageset(zone_pcp(zone,cpu), batch);
-#endif
-        }
         if (zone->present_pages)
-                printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%lu\n",
-                        zone->name, zone->present_pages, batch);
+                printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%u\n",
+                        zone->name, zone->present_pages,
+                                         zone_batchsize(zone));
 }
 
 __meminit int init_currently_empty_zone(struct zone *zone,
@@ -3424,6 +3393,61 @@ void __init free_bootmem_with_active_regions(int nid,
         }
 }
 
+int __init add_from_early_node_map(struct range *range, int az,
+                                   int nr_range, int nid)
+{
+        int i;
+        u64 start, end;
+
+        /* need to go over early_node_map to find out good range for node */
+        for_each_active_range_index_in_nid(i, nid) {
+                start = early_node_map[i].start_pfn;
+                end = early_node_map[i].end_pfn;
+                nr_range = add_range(range, az, nr_range, start, end);
+        }
+        return nr_range;
+}
+
+#ifdef CONFIG_NO_BOOTMEM
+void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
+                                        u64 goal, u64 limit)
+{
+        int i;
+        void *ptr;
+
+        /* need to go over early_node_map to find out good range for node */
+        for_each_active_range_index_in_nid(i, nid) {
+                u64 addr;
+                u64 ei_start, ei_last;
+
+                ei_last = early_node_map[i].end_pfn;
+                ei_last <<= PAGE_SHIFT;
+                ei_start = early_node_map[i].start_pfn;
+                ei_start <<= PAGE_SHIFT;
+                addr = find_early_area(ei_start, ei_last,
+                                         goal, limit, size, align);
+
+                if (addr == -1ULL)
+                        continue;
+
+#if 0
+                printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n",
+                                nid,
+                                ei_start, ei_last, goal, limit, size,
+                                align, addr);
+#endif
+
+                ptr = phys_to_virt(addr);
+                memset(ptr, 0, size);
+                reserve_early_without_check(addr, addr + size, "BOOTMEM");
+                return ptr;
+        }
+
+        return NULL;
+}
+#endif
+
+
 void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data)
 {
         int i;
@@ -3573,7 +3597,7 @@ static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  * then all holes in the requested range will be accounted for.
  */
-static unsigned long __meminit __absent_pages_in_range(int nid,
+unsigned long __meminit __absent_pages_in_range(int nid,
                                 unsigned long range_start_pfn,
                                 unsigned long range_end_pfn)
 {
@@ -3988,7 +4012,7 @@ void __init add_active_range(unsigned int nid, unsigned long start_pfn,
                 }
 
                 /* Merge backward if suitable */
-                if (start_pfn < early_node_map[i].end_pfn &&
+                if (start_pfn < early_node_map[i].start_pfn &&
                                 end_pfn >= early_node_map[i].start_pfn) {
                         early_node_map[i].start_pfn = start_pfn;
                         return;
@@ -4102,7 +4126,7 @@ static int __init cmp_node_active_region(const void *a, const void *b)
 }
 
 /* sort the node_map by start_pfn */
-static void __init sort_node_map(void)
+void __init sort_node_map(void)
 {
         sort(early_node_map, (size_t)nr_nodemap_entries,
                         sizeof(struct node_active_region),
@@ -4366,8 +4390,12 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
         for (i = 0; i < MAX_NR_ZONES; i++) {
                 if (i == ZONE_MOVABLE)
                         continue;
-                printk("  %-8s %0#10lx -> %0#10lx\n",
-                                zone_names[i],
+                printk("  %-8s ", zone_names[i]);
+                if (arch_zone_lowest_possible_pfn[i] ==
+                                arch_zone_highest_possible_pfn[i])
+                        printk("empty\n");
+                else
+                        printk("%0#10lx -> %0#10lx\n",
                                 arch_zone_lowest_possible_pfn[i],
                                 arch_zone_highest_possible_pfn[i]);
         }
@@ -4456,7 +4484,11 @@ void __init set_dma_reserve(unsigned long new_dma_reserve)
 }
 
 #ifndef CONFIG_NEED_MULTIPLE_NODES
-struct pglist_data __refdata contig_page_data = { .bdata = &bootmem_node_data[0] };
+struct pglist_data __refdata contig_page_data = {
+#ifndef CONFIG_NO_BOOTMEM
+ .bdata = &bootmem_node_data[0]
+#endif
+ };
 EXPORT_SYMBOL(contig_page_data);
 #endif
 
@@ -4799,10 +4831,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
         if (!write || (ret == -EINVAL))
                 return ret;
         for_each_populated_zone(zone) {
-                for_each_online_cpu(cpu) {
+                for_each_possible_cpu(cpu) {
                         unsigned long  high;
                         high = zone->present_pages / percpu_pagelist_fraction;
-                        setup_pagelist_highmark(zone_pcp(zone, cpu), high);
+                        setup_pagelist_highmark(
+                                per_cpu_ptr(zone->pageset, cpu), high);
                 }
         }
         return 0;
@@ -5002,23 +5035,65 @@ void set_pageblock_flags_group(struct page *page, unsigned long flags,
 int set_migratetype_isolate(struct page *page)
 {
         struct zone *zone;
-        unsigned long flags;
+        struct page *curr_page;
+        unsigned long flags, pfn, iter;
+        unsigned long immobile = 0;
+        struct memory_isolate_notify arg;
+        int notifier_ret;
         int ret = -EBUSY;
         int zone_idx;
 
         zone = page_zone(page);
         zone_idx = zone_idx(zone);
+
         spin_lock_irqsave(&zone->lock, flags);
+        if (get_pageblock_migratetype(page) == MIGRATE_MOVABLE ||
+            zone_idx == ZONE_MOVABLE) {
+                ret = 0;
+                goto out;
+        }
+
+        pfn = page_to_pfn(page);
+        arg.start_pfn = pfn;
+        arg.nr_pages = pageblock_nr_pages;
+        arg.pages_found = 0;
+
         /*
-         * In future, more migrate types will be able to be isolation target.
+         * It may be possible to isolate a pageblock even if the
+         * migratetype is not MIGRATE_MOVABLE. The memory isolation
+         * notifier chain is used by balloon drivers to return the
+         * number of pages in a range that are held by the balloon
+         * driver to shrink memory. If all the pages are accounted for
+         * by balloons, are free, or on the LRU, isolation can continue.
+         * Later, for example, when memory hotplug notifier runs, these
+         * pages reported as "can be isolated" should be isolated(freed)
+         * by the balloon driver through the memory notifier chain.
          */
-        if (get_pageblock_migratetype(page) != MIGRATE_MOVABLE &&
-            zone_idx != ZONE_MOVABLE)
+        notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
+        notifier_ret = notifier_to_errno(notifier_ret);
+        if (notifier_ret || !arg.pages_found)
                 goto out;
-        set_pageblock_migratetype(page, MIGRATE_ISOLATE);
-        move_freepages_block(zone, page, MIGRATE_ISOLATE);
-        ret = 0;
+
+        for (iter = pfn; iter < (pfn + pageblock_nr_pages); iter++) {
+                if (!pfn_valid_within(pfn))
+                        continue;
+
+                curr_page = pfn_to_page(iter);
+                if (!page_count(curr_page) || PageLRU(curr_page))
+                        continue;
+
+                immobile++;
+        }
+
+        if (arg.pages_found == immobile)
+                ret = 0;
+
 out:
+        if (!ret) {
+                set_pageblock_migratetype(page, MIGRATE_ISOLATE);
+                move_freepages_block(zone, page, MIGRATE_ISOLATE);
+        }
+
         spin_unlock_irqrestore(&zone->lock, flags);
         if (!ret)
                 drain_all_pages();
@@ -5085,3 +5160,101 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
         spin_unlock_irqrestore(&zone->lock, flags);
 }
 #endif
+
+#ifdef CONFIG_MEMORY_FAILURE
+bool is_free_buddy_page(struct page *page)
+{
+        struct zone *zone = page_zone(page);
+        unsigned long pfn = page_to_pfn(page);
+        unsigned long flags;
+        int order;
+
+        spin_lock_irqsave(&zone->lock, flags);
+        for (order = 0; order < MAX_ORDER; order++) {
+                struct page *page_head = page - (pfn & ((1 << order) - 1));
+
+                if (PageBuddy(page_head) && page_order(page_head) >= order)
+                        break;
+        }
+        spin_unlock_irqrestore(&zone->lock, flags);
+
+        return order < MAX_ORDER;
+}
+#endif
+
+static struct trace_print_flags pageflag_names[] = {
+        {1UL << PG_locked,              "locked"        },
+        {1UL << PG_error,               "error"         },
+        {1UL << PG_referenced,          "referenced"    },
+        {1UL << PG_uptodate,            "uptodate"      },
+        {1UL << PG_dirty,               "dirty"         },
+        {1UL << PG_lru,                 "lru"           },
+        {1UL << PG_active,              "active"        },
+        {1UL << PG_slab,                "slab"          },
+        {1UL << PG_owner_priv_1,        "owner_priv_1"  },
+        {1UL << PG_arch_1,              "arch_1"        },
+        {1UL << PG_reserved,            "reserved"      },
+        {1UL << PG_private,             "private"       },
+        {1UL << PG_private_2,           "private_2"     },
+        {1UL << PG_writeback,           "writeback"     },
+#ifdef CONFIG_PAGEFLAGS_EXTENDED
+        {1UL << PG_head,                "head"          },
+        {1UL << PG_tail,                "tail"          },
+#else
+        {1UL << PG_compound,            "compound"      },
+#endif
+        {1UL << PG_swapcache,           "swapcache"     },
+        {1UL << PG_mappedtodisk,        "mappedtodisk"  },
+        {1UL << PG_reclaim,             "reclaim"       },
+        {1UL << PG_buddy,               "buddy"         },
+        {1UL << PG_swapbacked,          "swapbacked"    },
+        {1UL << PG_unevictable,         "unevictable"   },
+#ifdef CONFIG_MMU
+        {1UL << PG_mlocked,             "mlocked"       },
+#endif
+#ifdef CONFIG_ARCH_USES_PG_UNCACHED
+        {1UL << PG_uncached,            "uncached"      },
+#endif
+#ifdef CONFIG_MEMORY_FAILURE
+        {1UL << PG_hwpoison,            "hwpoison"      },
+#endif
+        {-1UL,                          NULL            },
+};
+
+static void dump_page_flags(unsigned long flags)
+{
+        const char *delim = "";
+        unsigned long mask;
+        int i;
+
+        printk(KERN_ALERT "page flags: %#lx(", flags);
+
+        /* remove zone id */
+        flags &= (1UL << NR_PAGEFLAGS) - 1;
+
+        for (i = 0; pageflag_names[i].name && flags; i++) {
+
+                mask = pageflag_names[i].mask;
+                if ((flags & mask) != mask)
+                        continue;
+
+                flags &= ~mask;
+                printk("%s%s", delim, pageflag_names[i].name);
+                delim = "|";
+        }
+
+        /* check for left over flags */
+        if (flags)
+                printk("%s%#lx", delim, flags);
+
+        printk(")\n");
+}
+
+void dump_page(struct page *page)
+{
+        printk(KERN_ALERT
+               "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
+                page, page_count(page), page_mapcount(page),
+                page->mapping, page->index);
+        dump_page_flags(page->flags);
+}