1 files changed, 201 insertions, 116 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index c13ea7538891..bb90971182bd 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -558,7 +558,8 @@ static inline void __free_one_page(struct page *page,
                 if (page_is_guard(buddy)) {
                         clear_page_guard_flag(buddy);
                         set_page_private(page, 0);
-                        __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
+                        __mod_zone_freepage_state(zone, 1 << order,
+                                                  migratetype);
                 } else {
                         list_del(&buddy->lru);
                         zone->free_area[order].nr_free--;
@@ -597,17 +598,6 @@ out:
         zone->free_area[order].nr_free++;
 }
 
-/*
- * free_page_mlock() -- clean up attempts to free and mlocked() page.
- * Page should not be on lru, so no need to fix that up.
- * free_pages_check() will verify...
- */
-static inline void free_page_mlock(struct page *page)
-{
-        __dec_zone_page_state(page, NR_MLOCK);
-        __count_vm_event(UNEVICTABLE_MLOCKFREED);
-}
-
 static inline int free_pages_check(struct page *page)
 {
         if (unlikely(page_mapcount(page) |
@@ -668,12 +658,17 @@ static void free_pcppages_bulk(struct zone *zone, int count,
                         batch_free = to_free;
 
                 do {
+                        int mt; /* migratetype of the to-be-freed page */
+
                         page = list_entry(list->prev, struct page, lru);
                         /* must delete as __free_one_page list manipulates */
                         list_del(&page->lru);
+                        mt = get_freepage_migratetype(page);
                         /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
-                        __free_one_page(page, zone, 0, page_private(page));
-                        trace_mm_page_pcpu_drain(page, 0, page_private(page));
+                        __free_one_page(page, zone, 0, mt);
+                        trace_mm_page_pcpu_drain(page, 0, mt);
+                        if (is_migrate_cma(mt))
+                                __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
                 } while (--to_free && --batch_free && !list_empty(list));
         }
         __mod_zone_page_state(zone, NR_FREE_PAGES, count);
@@ -688,7 +683,8 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
         zone->pages_scanned = 0;
 
         __free_one_page(page, zone, order, migratetype);
-        __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
+        if (unlikely(migratetype != MIGRATE_ISOLATE))
+                __mod_zone_freepage_state(zone, 1 << order, migratetype);
         spin_unlock(&zone->lock);
 }
 
@@ -721,17 +717,16 @@ static bool free_pages_prepare(struct page *page, unsigned int order)
 static void __free_pages_ok(struct page *page, unsigned int order)
 {
         unsigned long flags;
-        int wasMlocked = __TestClearPageMlocked(page);
+        int migratetype;
 
         if (!free_pages_prepare(page, order))
                 return;
 
         local_irq_save(flags);
-        if (unlikely(wasMlocked))
-                free_page_mlock(page);
         __count_vm_events(PGFREE, 1 << order);
-        free_one_page(page_zone(page), page, order,
-                                        get_pageblock_migratetype(page));
+        migratetype = get_pageblock_migratetype(page);
+        set_freepage_migratetype(page, migratetype);
+        free_one_page(page_zone(page), page, order, migratetype);
         local_irq_restore(flags);
 }
 
@@ -811,7 +806,8 @@ static inline void expand(struct zone *zone, struct page *page,
                         set_page_guard_flag(&page[size]);
                         set_page_private(&page[size], high);
                         /* Guard pages are not available for any usage */
-                        __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << high));
+                        __mod_zone_freepage_state(zone, -(1 << high),
+                                                  migratetype);
                         continue;
                 }
 #endif
@@ -915,7 +911,7 @@ static int fallbacks[MIGRATE_TYPES][4] = {
  * Note that start_page and end_pages are not aligned on a pageblock
  * boundary. If alignment is required, use move_freepages_block()
  */
-static int move_freepages(struct zone *zone,
+int move_freepages(struct zone *zone,
                           struct page *start_page, struct page *end_page,
                           int migratetype)
 {
@@ -951,6 +947,7 @@ static int move_freepages(struct zone *zone,
                 order = page_order(page);
                 list_move(&page->lru,
                           &zone->free_area[order].free_list[migratetype]);
+                set_freepage_migratetype(page, migratetype);
                 page += 1 << order;
                 pages_moved += 1 << order;
         }
@@ -1135,8 +1132,11 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                         if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE)
                                 mt = migratetype;
                 }
-                set_page_private(page, mt);
+                set_freepage_migratetype(page, mt);
                 list = &page->lru;
+                if (is_migrate_cma(mt))
+                        __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
+                                              -(1 << order));
         }
         __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
         spin_unlock(&zone->lock);
@@ -1296,16 +1296,13 @@ void free_hot_cold_page(struct page *page, int cold)
         struct per_cpu_pages *pcp;
         unsigned long flags;
         int migratetype;
-        int wasMlocked = __TestClearPageMlocked(page);
 
         if (!free_pages_prepare(page, 0))
                 return;
 
         migratetype = get_pageblock_migratetype(page);
-        set_page_private(page, migratetype);
+        set_freepage_migratetype(page, migratetype);
         local_irq_save(flags);
-        if (unlikely(wasMlocked))
-                free_page_mlock(page);
         __count_vm_event(PGFREE);
 
         /*
@@ -1380,20 +1377,16 @@ void split_page(struct page *page, unsigned int order)
 }
 
 /*
- * Similar to split_page except the page is already free. As this is only
- * being used for migration, the migratetype of the block also changes.
- * As this is called with interrupts disabled, the caller is responsible
- * for calling arch_alloc_page() and kernel_map_page() after interrupts
- * are enabled.
- *
- * Note: this is probably too low level an operation for use in drivers.
- * Please consult with lkml before using this in your driver.
+ * Similar to the split_page family of functions except that the page
+ * required at the given order and being isolated now to prevent races
+ * with parallel allocators
  */
-int split_free_page(struct page *page)
+int capture_free_page(struct page *page, int alloc_order, int migratetype)
 {
         unsigned int order;
         unsigned long watermark;
         struct zone *zone;
+        int mt;
 
         BUG_ON(!PageBuddy(page));
 
@@ -1409,12 +1402,16 @@ int split_free_page(struct page *page)
         list_del(&page->lru);
         zone->free_area[order].nr_free--;
         rmv_page_order(page);
-        __mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order));
 
-        /* Split into individual pages */
-        set_page_refcounted(page);
-        split_page(page, order);
+        mt = get_pageblock_migratetype(page);
+        if (unlikely(mt != MIGRATE_ISOLATE))
+                __mod_zone_freepage_state(zone, -(1UL << order), mt);
 
+        if (alloc_order != order)
+                expand(zone, page, alloc_order, order,
+                        &zone->free_area[order], migratetype);
+
+        /* Set the pageblock if the captured page is at least a pageblock */
         if (order >= pageblock_order - 1) {
                 struct page *endpage = page + (1 << order) - 1;
                 for (; page < endpage; page += pageblock_nr_pages) {
@@ -1425,7 +1422,35 @@ int split_free_page(struct page *page)
                 }
         }
 
-        return 1 << order;
+        return 1UL << order;
+}
+
+/*
+ * Similar to split_page except the page is already free. As this is only
+ * being used for migration, the migratetype of the block also changes.
+ * As this is called with interrupts disabled, the caller is responsible
+ * for calling arch_alloc_page() and kernel_map_page() after interrupts
+ * are enabled.
+ *
+ * Note: this is probably too low level an operation for use in drivers.
+ * Please consult with lkml before using this in your driver.
+ */
+int split_free_page(struct page *page)
+{
+        unsigned int order;
+        int nr_pages;
+
+        BUG_ON(!PageBuddy(page));
+        order = page_order(page);
+
+        nr_pages = capture_free_page(page, order, 0);
+        if (!nr_pages)
+                return 0;
+
+        /* Split into individual pages */
+        set_page_refcounted(page);
+        split_page(page, order);
+        return nr_pages;
 }
 
 /*
@@ -1484,7 +1509,8 @@ again:
                 spin_unlock(&zone->lock);
                 if (!page)
                         goto failed;
-                __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
+                __mod_zone_freepage_state(zone, -(1 << order),
+                                          get_pageblock_migratetype(page));
         }
 
         __count_zone_vm_events(PGALLOC, zone, 1 << order);
@@ -1501,19 +1527,6 @@ failed:
         return NULL;
 }
 
-/* The ALLOC_WMARK bits are used as an index to zone->watermark */
-#define ALLOC_WMARK_MIN         WMARK_MIN
-#define ALLOC_WMARK_LOW         WMARK_LOW
-#define ALLOC_WMARK_HIGH        WMARK_HIGH
-#define ALLOC_NO_WATERMARKS     0x04 /* don't check watermarks at all */
-
-/* Mask to get the watermark bits */
-#define ALLOC_WMARK_MASK        (ALLOC_NO_WATERMARKS-1)
-
-#define ALLOC_HARDER            0x10 /* try to alloc harder */
-#define ALLOC_HIGH              0x20 /* __GFP_HIGH set */
-#define ALLOC_CPUSET            0x40 /* check for correct cpuset */
-
 #ifdef CONFIG_FAIL_PAGE_ALLOC
 
 static struct {
@@ -1608,7 +1621,11 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
                 min -= min / 2;
         if (alloc_flags & ALLOC_HARDER)
                 min -= min / 4;
-
+#ifdef CONFIG_CMA
+        /* If allocation can't use CMA areas don't use free CMA pages */
+        if (!(alloc_flags & ALLOC_CMA))
+                free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
+#endif
         if (free_pages <= min + lowmem_reserve)
                 return false;
         for (o = 0; o < order; o++) {
@@ -1782,6 +1799,22 @@ static void zlc_clear_zones_full(struct zonelist *zonelist)
         bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
 }
 
+static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
+{
+        return node_isset(local_zone->node, zone->zone_pgdat->reclaim_nodes);
+}
+
+static void __paginginit init_zone_allows_reclaim(int nid)
+{
+        int i;
+
+        for_each_online_node(i)
+                if (node_distance(nid, i) <= RECLAIM_DISTANCE) {
+                        node_set(i, NODE_DATA(nid)->reclaim_nodes);
+                        zone_reclaim_mode = 1;
+                }
+}
+
 #else   /* CONFIG_NUMA */
 
 static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
@@ -1802,6 +1835,15 @@ static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
 static void zlc_clear_zones_full(struct zonelist *zonelist)
 {
 }
+
+static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
+{
+        return true;
+}
+
+static inline void init_zone_allows_reclaim(int nid)
+{
+}
 #endif  /* CONFIG_NUMA */
 
 /*
@@ -1886,7 +1928,8 @@ zonelist_scan:
                                 did_zlc_setup = 1;
                         }
 
-                        if (zone_reclaim_mode == 0)
+                        if (zone_reclaim_mode == 0 ||
+                            !zone_allows_reclaim(preferred_zone, zone))
                                 goto this_zone_full;
 
                         /*
@@ -2105,7 +2148,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
         bool *contended_compaction, bool *deferred_compaction,
         unsigned long *did_some_progress)
 {
-        struct page *page;
+        struct page *page = NULL;
 
         if (!order)
                 return NULL;
@@ -2118,10 +2161,16 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
         current->flags |= PF_MEMALLOC;
         *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
                                                 nodemask, sync_migration,
-                                                contended_compaction);
+                                                contended_compaction, &page);
         current->flags &= ~PF_MEMALLOC;
-        if (*did_some_progress != COMPACT_SKIPPED) {
 
+        /* If compaction captured a page, prep and use it */
+        if (page) {
+                prep_new_page(page, order, gfp_mask);
+                goto got_page;
+        }
+
+        if (*did_some_progress != COMPACT_SKIPPED) {
                 /* Page migration frees to the PCP lists but we want merging */
                 drain_pages(get_cpu());
                 put_cpu();
@@ -2131,6 +2180,8 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
                                 alloc_flags & ~ALLOC_NO_WATERMARKS,
                                 preferred_zone, migratetype);
                 if (page) {
+got_page:
+                        preferred_zone->compact_blockskip_flush = false;
                         preferred_zone->compact_considered = 0;
                         preferred_zone->compact_defer_shift = 0;
                         if (order >= preferred_zone->compact_order_failed)
@@ -2315,7 +2366,10 @@ gfp_to_alloc_flags(gfp_t gfp_mask)
                                  unlikely(test_thread_flag(TIF_MEMDIE))))
                         alloc_flags |= ALLOC_NO_WATERMARKS;
         }
-
+#ifdef CONFIG_CMA
+        if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
+                alloc_flags |= ALLOC_CMA;
+#endif
         return alloc_flags;
 }
 
@@ -2362,9 +2416,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
                 goto nopage;
 
 restart:
-        if (!(gfp_mask & __GFP_NO_KSWAPD))
-                wake_all_kswapd(order, zonelist, high_zoneidx,
-                                                zone_idx(preferred_zone));
+        wake_all_kswapd(order, zonelist, high_zoneidx,
+                                        zone_idx(preferred_zone));
 
         /*
          * OK, we're below the kswapd watermark and have kicked background
@@ -2441,7 +2494,7 @@ rebalance:
          * system then fail the allocation instead of entering direct reclaim.
          */
         if ((deferred_compaction || contended_compaction) &&
-                                                (gfp_mask & __GFP_NO_KSWAPD))
+            (gfp_mask & (__GFP_MOVABLE|__GFP_REPEAT)) == __GFP_MOVABLE)
                 goto nopage;
 
         /* Try direct reclaim and then allocating */
@@ -2541,6 +2594,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
         struct page *page = NULL;
         int migratetype = allocflags_to_migratetype(gfp_mask);
         unsigned int cpuset_mems_cookie;
+        int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET;
 
         gfp_mask &= gfp_allowed_mask;
 
@@ -2569,9 +2623,13 @@ retry_cpuset:
         if (!preferred_zone)
                 goto out;
 
+#ifdef CONFIG_CMA
+        if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
+                alloc_flags |= ALLOC_CMA;
+#endif
         /* First allocation attempt */
         page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
-                        zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
+                        zonelist, high_zoneidx, alloc_flags,
                         preferred_zone, migratetype);
         if (unlikely(!page))
                 page = __alloc_pages_slowpath(gfp_mask, order,
@@ -2852,7 +2910,8 @@ void show_free_areas(unsigned int filter)
                 " unevictable:%lu"
                 " dirty:%lu writeback:%lu unstable:%lu\n"
                 " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
-                " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n",
+                " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
+                " free_cma:%lu\n",
                 global_page_state(NR_ACTIVE_ANON),
                 global_page_state(NR_INACTIVE_ANON),
                 global_page_state(NR_ISOLATED_ANON),
@@ -2869,7 +2928,8 @@ void show_free_areas(unsigned int filter)
                 global_page_state(NR_FILE_MAPPED),
                 global_page_state(NR_SHMEM),
                 global_page_state(NR_PAGETABLE),
-                global_page_state(NR_BOUNCE));
+                global_page_state(NR_BOUNCE),
+                global_page_state(NR_FREE_CMA_PAGES));
 
         for_each_populated_zone(zone) {
                 int i;
@@ -2901,6 +2961,7 @@ void show_free_areas(unsigned int filter)
                         " pagetables:%lukB"
                         " unstable:%lukB"
                         " bounce:%lukB"
+                        " free_cma:%lukB"
                         " writeback_tmp:%lukB"
                         " pages_scanned:%lu"
                         " all_unreclaimable? %s"
@@ -2930,6 +2991,7 @@ void show_free_areas(unsigned int filter)
                         K(zone_page_state(zone, NR_PAGETABLE)),
                         K(zone_page_state(zone, NR_UNSTABLE_NFS)),
                         K(zone_page_state(zone, NR_BOUNCE)),
+                        K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
                         K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
                         zone->pages_scanned,
                         (zone->all_unreclaimable ? "yes" : "no")
@@ -3328,21 +3390,13 @@ static void build_zonelists(pg_data_t *pgdat)
         j = 0;
 
         while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
-                int distance = node_distance(local_node, node);
-
-                /*
-                 * If another node is sufficiently far away then it is better
-                 * to reclaim pages in a zone before going off node.
-                 */
-                if (distance > RECLAIM_DISTANCE)
-                        zone_reclaim_mode = 1;
-
                 /*
                  * We don't want to pressure a particular node.
                  * So adding penalty to the first node in same
                  * distance group to make it round-robin.
                  */
-                if (distance != node_distance(local_node, prev_node))
+                if (node_distance(local_node, node) !=
+                    node_distance(local_node, prev_node))
                         node_load[node] = load;
 
                 prev_node = node;
@@ -4438,11 +4492,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 
                 zone->spanned_pages = size;
                 zone->present_pages = realsize;
-#if defined CONFIG_COMPACTION || defined CONFIG_CMA
-                zone->compact_cached_free_pfn = zone->zone_start_pfn +
-                                                zone->spanned_pages;
-                zone->compact_cached_free_pfn &= ~(pageblock_nr_pages-1);
-#endif
 #ifdef CONFIG_NUMA
                 zone->node = nid;
                 zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
@@ -4521,6 +4570,7 @@ void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
 
         pgdat->node_id = nid;
         pgdat->node_start_pfn = node_start_pfn;
+        init_zone_allows_reclaim(nid);
         calculate_node_totalpages(pgdat, zones_size, zholes_size);
 
         alloc_node_mem_map(pgdat);
@@ -4879,7 +4929,7 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
                                zone_movable_pfn[i] << PAGE_SHIFT);
         }
 
-        /* Print out the early_node_map[] */
+        /* Print out the early node map */
         printk("Early memory node ranges\n");
         for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
                 printk("  node %3d: [mem %#010lx-%#010lx]\n", nid,
@@ -5619,47 +5669,28 @@ static unsigned long pfn_max_align_up(unsigned long pfn)
                                 pageblock_nr_pages));
 }
 
-static struct page *
-__alloc_contig_migrate_alloc(struct page *page, unsigned long private,
-                             int **resultp)
-{
-        gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE;
-
-        if (PageHighMem(page))
-                gfp_mask |= __GFP_HIGHMEM;
-
-        return alloc_page(gfp_mask);
-}
-
 /* [start, end) must belong to a single zone. */
-static int __alloc_contig_migrate_range(unsigned long start, unsigned long end)
+static int __alloc_contig_migrate_range(struct compact_control *cc,
+                                        unsigned long start, unsigned long end)
 {
         /* This function is based on compact_zone() from compaction.c. */
-
+        unsigned long nr_reclaimed;
         unsigned long pfn = start;
         unsigned int tries = 0;
         int ret = 0;
 
-        struct compact_control cc = {
-                .nr_migratepages = 0,
-                .order = -1,
-                .zone = page_zone(pfn_to_page(start)),
-                .sync = true,
-        };
-        INIT_LIST_HEAD(&cc.migratepages);
-
         migrate_prep_local();
 
-        while (pfn < end || !list_empty(&cc.migratepages)) {
+        while (pfn < end || !list_empty(&cc->migratepages)) {
                 if (fatal_signal_pending(current)) {
                         ret = -EINTR;
                         break;
                 }
 
-                if (list_empty(&cc.migratepages)) {
-                        cc.nr_migratepages = 0;
-                        pfn = isolate_migratepages_range(cc.zone, &cc,
-                                                         pfn, end);
+                if (list_empty(&cc->migratepages)) {
+                        cc->nr_migratepages = 0;
+                        pfn = isolate_migratepages_range(cc->zone, cc,
+                                                         pfn, end, true);
                         if (!pfn) {
                                 ret = -EINTR;
                                 break;
@@ -5670,12 +5701,16 @@ static int __alloc_contig_migrate_range(unsigned long start, unsigned long end)
                         break;
                 }
 
-                ret = migrate_pages(&cc.migratepages,
-                                    __alloc_contig_migrate_alloc,
+                nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
+                                                        &cc->migratepages);
+                cc->nr_migratepages -= nr_reclaimed;
+
+                ret = migrate_pages(&cc->migratepages,
+                                    alloc_migrate_target,
                                     0, false, MIGRATE_SYNC);
         }
 
-        putback_lru_pages(&cc.migratepages);
+        putback_lru_pages(&cc->migratepages);
         return ret > 0 ? 0 : ret;
 }
 
@@ -5754,6 +5789,15 @@ int alloc_contig_range(unsigned long start, unsigned long end,
         unsigned long outer_start, outer_end;
         int ret = 0, order;
 
+        struct compact_control cc = {
+                .nr_migratepages = 0,
+                .order = -1,
+                .zone = page_zone(pfn_to_page(start)),
+                .sync = true,
+                .ignore_skip_hint = true,
+        };
+        INIT_LIST_HEAD(&cc.migratepages);
+
         /*
          * What we do here is we mark all pageblocks in range as
          * MIGRATE_ISOLATE.  Because pageblock and max order pages may
@@ -5783,7 +5827,7 @@ int alloc_contig_range(unsigned long start, unsigned long end,
         if (ret)
                 goto done;
 
-        ret = __alloc_contig_migrate_range(start, end);
+        ret = __alloc_contig_migrate_range(&cc, start, end);
         if (ret)
                 goto done;
 
@@ -5832,7 +5876,7 @@ int alloc_contig_range(unsigned long start, unsigned long end,
         __reclaim_pages(zone, GFP_HIGHUSER_MOVABLE, end-start);
 
         /* Grab isolated pages from freelists. */
-        outer_end = isolate_freepages_range(outer_start, end);
+        outer_end = isolate_freepages_range(&cc, outer_start, end);
         if (!outer_end) {
                 ret = -EBUSY;
                 goto done;
@@ -5874,6 +5918,7 @@ static int __meminit __zone_pcp_update(void *data)
                 local_irq_save(flags);
                 if (pcp->count > 0)
                         free_pcppages_bulk(zone, pcp->count, pcp);
+                drain_zonestat(zone, pset);
                 setup_pageset(pset, batch);
                 local_irq_restore(flags);
         }
@@ -5890,10 +5935,16 @@ void __meminit zone_pcp_update(struct zone *zone)
 void zone_pcp_reset(struct zone *zone)
 {
         unsigned long flags;
+        int cpu;
+        struct per_cpu_pageset *pset;
 
         /* avoid races with drain_pages()  */
         local_irq_save(flags);
         if (zone->pageset != &boot_pageset) {
+                for_each_online_cpu(cpu) {
+                        pset = per_cpu_ptr(zone->pageset, cpu);
+                        drain_zonestat(zone, pset);
+                }
                 free_percpu(zone->pageset);
                 zone->pageset = &boot_pageset;
         }
@@ -6047,3 +6098,37 @@ void dump_page(struct page *page)
         dump_page_flags(page->flags);
         mem_cgroup_print_bad_page(page);
 }
+
+/* reset zone->present_pages */
+void reset_zone_present_pages(void)
+{
+        struct zone *z;
+        int i, nid;
+
+        for_each_node_state(nid, N_HIGH_MEMORY) {
+                for (i = 0; i < MAX_NR_ZONES; i++) {
+                        z = NODE_DATA(nid)->node_zones + i;
+                        z->present_pages = 0;
+                }
+        }
+}
+
+/* calculate zone's present pages in buddy system */
+void fixup_zone_present_pages(int nid, unsigned long start_pfn,
+                                unsigned long end_pfn)
+{
+        struct zone *z;
+        unsigned long zone_start_pfn, zone_end_pfn;
+        int i;
+
+        for (i = 0; i < MAX_NR_ZONES; i++) {
+                z = NODE_DATA(nid)->node_zones + i;
+                zone_start_pfn = z->zone_start_pfn;
+                zone_end_pfn = zone_start_pfn + z->spanned_pages;
+
+                /* if the two regions intersect */
+                if (!(zone_start_pfn >= end_pfn || zone_end_pfn <= start_pfn))
+                        z->present_pages += min(end_pfn, zone_end_pfn) -
+                                            max(start_pfn, zone_start_pfn);
+        }
+}